Rename packages to improve readability. (#654)

* Rename packages to improve readability.

This fixes #651

Changes:
- internal/pkgecosystem -> internal/pkgmanager
- pkg/api -> pkg/api/pkgecosystem + pkg/api/analysisrun.DynamicPhase
- pkg/result -> pkg/api/analysisrun
- pkg/notification -> pkg/api/notification
- pkg/pkgidentifier -> pkg/api/analysisrun.Key

Note - this is a breaking change.

Signed-off-by: Caleb Brown <[email protected]>

* Make it easier to use Ecosystem in a flag.

Signed-off-by: Caleb Brown <[email protected]>

* Add missing space.

Signed-off-by: Caleb Brown <[email protected]>

* Make the ecosystem flag usage even more flexible.

Signed-off-by: Caleb Brown <[email protected]>

* Remove unused FlagUsage

Signed-off-by: Caleb Brown <[email protected]>

* Tweak again the approach to returning a list of ecosystems.

Signed-off-by: Caleb Brown <[email protected]>

---------

Signed-off-by: Caleb Brown <[email protected]>

Author: calebbrown

cmd/analyze/main.go

@@ -10,19 +10,19 @@ import (
 
 	"github.com/ossf/package-analysis/internal/analysis"
 	"github.com/ossf/package-analysis/internal/log"
-	"github.com/ossf/package-analysis/internal/pkgecosystem"
+	"github.com/ossf/package-analysis/internal/pkgmanager"
 	"github.com/ossf/package-analysis/internal/resultstore"
 	"github.com/ossf/package-analysis/internal/sandbox"
 	"github.com/ossf/package-analysis/internal/staticanalysis"
 	"github.com/ossf/package-analysis/internal/utils"
 	"github.com/ossf/package-analysis/internal/worker"
-	"github.com/ossf/package-analysis/pkg/api"
+	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
 var (
 	pkgName             = flag.String("package", "", "package name")
 	localPkg            = flag.String("local", "", "local package path")
-	ecosystem           = flag.String("ecosystem", "", "ecosystem (npm, pypi, or rubygems)")
+	ecosystem           pkgecosystem.Ecosystem
 	version             = flag.String("version", "", "version")
 	noPull              = flag.Bool("nopull", false, "disables pulling down sandbox images")
 	imageTag            = flag.String("image-tag", "", "set image tag for analysis sandboxes")
@@ -78,7 +78,7 @@ func makeSandboxOptions(mode analysis.Mode) []sandbox.Option {
 	return sbOpts
 }
 
-func dynamicAnalysis(pkg *pkgecosystem.Pkg) {
+func dynamicAnalysis(pkg *pkgmanager.Pkg) {
 	if !*offline {
 		sandbox.InitNetwork()
 	}
@@ -116,7 +116,7 @@ func dynamicAnalysis(pkg *pkgecosystem.Pkg) {
 	}
 }
 
-func staticAnalysis(pkg *pkgecosystem.Pkg) {
+func staticAnalysis(pkg *pkgmanager.Pkg) {
 	if !*offline {
 		sandbox.InitNetwork()
 	}
@@ -143,6 +143,8 @@ func staticAnalysis(pkg *pkgecosystem.Pkg) {
 func main() {
 	log.Initialize(os.Getenv("LOGGER_ENV"))
 
+	flag.TextVar(&ecosystem, "ecosystem", pkgecosystem.None, fmt.Sprintf("package ecosystem. Can be %s", pkgecosystem.SupportedEcosystemsStrings))
+
 	analysisMode.InitFlag()
 	flag.Parse()
 
@@ -156,15 +158,15 @@ func main() {
 		return
 	}
 
-	if *ecosystem == "" {
+	if ecosystem == pkgecosystem.None {
 		flag.Usage()
 		return
 	}
 
-	manager := pkgecosystem.Manager(api.Ecosystem(*ecosystem), *combinedSandbox)
+	manager := pkgmanager.Manager(ecosystem, *combinedSandbox)
 	if manager == nil {
 		log.Panic("Unsupported pkg manager",
-			log.Label("ecosystem", *ecosystem))
+			log.Label("ecosystem", string(ecosystem)))
 	}
 
 	if *pkgName == "" {
@@ -183,12 +185,12 @@ func main() {
 		runMode[mode] = true
 	}
 
-	worker.LogRequest(*ecosystem, *pkgName, *version, *localPkg, "")
+	worker.LogRequest(ecosystem, *pkgName, *version, *localPkg, "")
 
 	pkg, err := worker.ResolvePkg(manager, *pkgName, *version, *localPkg)
 	if err != nil {
 		log.Panic("Error resolving package",
-			log.Label("ecosystem", *ecosystem),
+			log.Label("ecosystem", ecosystem.String()),
 			"name", *pkgName,
 			"error", err)
 	}

cmd/scheduler/main.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/ossf/package-analysis/cmd/scheduler/proxy"
 	"github.com/ossf/package-analysis/internal/log"
+	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
 const (
@@ -31,7 +32,7 @@ type ManagerConfig struct {
 	ExcludeVersions []*regexp.Regexp
 
 	// Ecosystem is the internal name of the ecosystem.
-	Ecosystem string
+	Ecosystem pkgecosystem.Ecosystem
 }
 
 func (m *ManagerConfig) SkipVersion(version string) bool {
@@ -53,14 +54,14 @@ func (m *ManagerConfig) SkipVersion(version string) bool {
 // analyze. It is a map from ossf/package-feeds package types, to a
 // config for the package manager's feed.
 var supportedPkgManagers = map[string]*ManagerConfig{
-	"npm":      {Ecosystem: "npm"},
-	"pypi":     {Ecosystem: "pypi"},
-	"rubygems": {Ecosystem: "rubygems"},
+	"npm":      {Ecosystem: pkgecosystem.NPM},
+	"pypi":     {Ecosystem: pkgecosystem.PyPI},
+	"rubygems": {Ecosystem: pkgecosystem.RubyGems},
 	"packagist": {
-		Ecosystem:       "packagist",
+		Ecosystem:       pkgecosystem.Packagist,
 		ExcludeVersions: []*regexp.Regexp{regexp.MustCompile(`^dev-`), regexp.MustCompile(`\.x-dev$`)},
 	},
-	"crates": {Ecosystem: "crates.io"},
+	"crates": {Ecosystem: pkgecosystem.CratesIO},
 }
 
 func main() {
@@ -123,7 +124,7 @@ func listenLoop(subURL, topicURL string) error {
 			Body: []byte{},
 			Metadata: map[string]string{
 				"name":      pkg.Name,
-				"ecosystem": config.Ecosystem,
+				"ecosystem": config.Ecosystem.String(),
 				"version":   pkg.Version,
 			},
 		}, nil

cmd/worker/main.go

@@ -23,13 +23,13 @@ import (
 	"github.com/ossf/package-analysis/internal/analysis"
 	"github.com/ossf/package-analysis/internal/log"
 	"github.com/ossf/package-analysis/internal/notification"
-	"github.com/ossf/package-analysis/internal/pkgecosystem"
+	"github.com/ossf/package-analysis/internal/pkgmanager"
 	"github.com/ossf/package-analysis/internal/resultstore"
 	"github.com/ossf/package-analysis/internal/sandbox"
 	"github.com/ossf/package-analysis/internal/staticanalysis"
 	"github.com/ossf/package-analysis/internal/worker"
-	"github.com/ossf/package-analysis/pkg/api"
-	"github.com/ossf/package-analysis/pkg/result"
+	"github.com/ossf/package-analysis/pkg/api/analysisrun"
+	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
 const (
@@ -80,7 +80,7 @@ func copyPackageToLocalFile(ctx context.Context, packagesBucket *blob.Bucket, bu
 	return fmt.Sprintf(localPkgPathFmt, path.Base(bucketPath)), f, nil
 }
 
-func saveResults(ctx context.Context, pkg *pkgecosystem.Pkg, dest resultBucketPaths, dynamicResults result.DynamicAnalysisResults, staticResults result.StaticAnalysisResults) error {
+func saveResults(ctx context.Context, pkg *pkgmanager.Pkg, dest resultBucketPaths, dynamicResults analysisrun.DynamicAnalysisResults, staticResults analysisrun.StaticAnalysisResults) error {
 	if dest.dynamicAnalysis != "" {
 		err := resultstore.New(dest.dynamicAnalysis, resultstore.ConstructPath()).Save(ctx, pkg, dynamicResults.StraceSummary)
 		if err != nil {
@@ -111,18 +111,18 @@ func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blo
 		return nil
 	}
 
-	ecosystem := msg.Metadata["ecosystem"]
+	ecosystem := pkgecosystem.Ecosystem(msg.Metadata["ecosystem"])
 	if ecosystem == "" {
 		log.Warn("ecosystem is empty",
 			"name", name)
 		msg.Ack()
 		return nil
 	}
 
-	manager := pkgecosystem.Manager(api.Ecosystem(ecosystem), false)
+	manager := pkgmanager.Manager(ecosystem, false)
 	if manager == nil {
 		log.Warn("Unsupported pkg manager",
-			log.Label("ecosystem", ecosystem),
+			log.Label("ecosystem", ecosystem.String()),
 			"name", name)
 		msg.Ack()
 		return nil
@@ -165,7 +165,7 @@ func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blo
 	pkg, err := worker.ResolvePkg(manager, name, version, localPkgPath)
 	if err != nil {
 		log.Error("Error resolving package",
-			log.Label("ecosystem", ecosystem),
+			log.Label("ecosystem", ecosystem.String()),
 			log.Label("name", name),
 			"error", err)
 		return err
@@ -176,7 +176,7 @@ func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blo
 		return err
 	}
 
-	var staticResults result.StaticAnalysisResults
+	var staticResults analysisrun.StaticAnalysisResults
 	if resultsBuckets.staticAnalysis != "" {
 		staticResults, _, err = worker.RunStaticAnalysis(pkg, staticSandboxOpts, staticanalysis.All)
 		if err != nil {

internal/dynamicanalysis/analysis.go

@@ -9,7 +9,7 @@ import (
 	"github.com/ossf/package-analysis/internal/packetcapture"
 	"github.com/ossf/package-analysis/internal/sandbox"
 	"github.com/ossf/package-analysis/internal/strace"
-	"github.com/ossf/package-analysis/pkg/result"
+	"github.com/ossf/package-analysis/pkg/api/analysisrun"
 )
 
 const (
@@ -18,12 +18,12 @@ const (
 )
 
 type Result struct {
-	StraceSummary result.StraceSummary
-	FileWrites    result.FileWrites
+	StraceSummary analysisrun.StraceSummary
+	FileWrites    analysisrun.FileWrites
 }
 
 var resultError = &Result{
-	StraceSummary: result.StraceSummary{
+	StraceSummary: analysisrun.StraceSummary{
 		Status: analysis.StatusErrorOther,
 	},
 }
@@ -67,7 +67,7 @@ func Run(sb sandbox.Sandbox, args []string) (*Result, error) {
 	}
 
 	analysisResult := Result{
-		StraceSummary: result.StraceSummary{
+		StraceSummary: analysisrun.StraceSummary{
 			Status: analysis.StatusForRunResult(r),
 			Stdout: lastLines(r.Stdout(), maxOutputLines, maxOutputBytes),
 			Stderr: lastLines(r.Stderr(), maxOutputLines, maxOutputBytes),
@@ -79,36 +79,42 @@ func Run(sb sandbox.Sandbox, args []string) (*Result, error) {
 
 func (d *Result) setData(straceResult *strace.Result, dns *dnsanalyzer.DNSAnalyzer) {
 	for _, f := range straceResult.Files() {
-		d.StraceSummary.Files = append(d.StraceSummary.Files, result.FileResult{
+		d.StraceSummary.Files = append(d.StraceSummary.Files, analysisrun.FileResult{
 			Path:   f.Path,
 			Read:   f.Read,
 			Write:  f.Write,
 			Delete: f.Delete,
 		})
 		if len(f.WriteInfo) > 0 {
-			d.FileWrites = append(d.FileWrites, result.FileWriteResult{f.Path, f.WriteInfo})
+			w := analysisrun.FileWriteResult{Path: f.Path}
+			for _, wi := range f.WriteInfo {
+				w.WriteInfo = append(w.WriteInfo, analysisrun.WriteInfo{
+					BytesWritten: wi.BytesWritten,
+				})
+			}
+			d.FileWrites = append(d.FileWrites, w)
 		}
 	}
 
 	for _, s := range straceResult.Sockets() {
-		d.StraceSummary.Sockets = append(d.StraceSummary.Sockets, result.SocketResult{
+		d.StraceSummary.Sockets = append(d.StraceSummary.Sockets, analysisrun.SocketResult{
 			Address:   s.Address,
 			Port:      s.Port,
 			Hostnames: dns.Hostnames(s.Address),
 		})
 	}
 
 	for _, c := range straceResult.Commands() {
-		d.StraceSummary.Commands = append(d.StraceSummary.Commands, result.CommandResult{
+		d.StraceSummary.Commands = append(d.StraceSummary.Commands, analysisrun.CommandResult{
 			Command:     c.Command,
 			Environment: c.Env,
 		})
 	}
 
 	for dnsClass, queries := range dns.Questions() {
-		c := result.DnsResult{Class: dnsClass}
+		c := analysisrun.DNSResult{Class: dnsClass}
 		for host, types := range queries {
-			c.Queries = append(c.Queries, result.DnsQueries{
+			c.Queries = append(c.Queries, analysisrun.DNSQueries{
 				Hostname: host,
 				Types:    types,
 			})

internal/notification/notification.go

@@ -7,13 +7,14 @@ import (
 
 	"gocloud.dev/pubsub"
 
-	"github.com/ossf/package-analysis/pkg/notification"
-	"github.com/ossf/package-analysis/pkg/pkgidentifier"
+	"github.com/ossf/package-analysis/pkg/api/analysisrun"
+	"github.com/ossf/package-analysis/pkg/api/notification"
+	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
-func PublishAnalysisCompletion(ctx context.Context, notificationTopic *pubsub.Topic, name, version, ecosystem string) error {
-	pkgDetails := pkgidentifier.PkgIdentifier{Name: name, Version: version, Ecosystem: ecosystem}
-	notificationMsg, err := json.Marshal(notification.AnalysisCompletion{Package: pkgDetails})
+func PublishAnalysisCompletion(ctx context.Context, notificationTopic *pubsub.Topic, name, version string, ecosystem pkgecosystem.Ecosystem) error {
+	k := analysisrun.Key{Name: name, Version: version, Ecosystem: ecosystem}
+	notificationMsg, err := json.Marshal(notification.AnalysisRunComplete{Key: k})
 	if err != nil {
 		return fmt.Errorf("failed to encode completion notification: %w", err)
 	}

internal/pkgecosystem/crates.io.go renamed to internal/pkgmanager/crates.io.go

@@ -1,11 +1,12 @@
-package pkgecosystem
+package pkgmanager
 
 import (
 	"encoding/json"
 	"fmt"
 	"net/http"
 
-	"github.com/ossf/package-analysis/pkg/api"
+	"github.com/ossf/package-analysis/pkg/api/analysisrun"
+	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
 type cratesJSON struct {
@@ -32,21 +33,21 @@ func getCratesLatest(pkg string) (string, error) {
 }
 
 var cratesPkgManager = PkgManager{
-	ecosystem:     api.EcosystemCratesIO,
+	ecosystem:     pkgecosystem.CratesIO,
 	image:         "gcr.io/ossf-malware-analysis/crates.io",
 	command:       "/usr/local/bin/analyze.py",
 	latestVersion: getCratesLatest,
-	runPhases: []api.RunPhase{
-		api.RunPhaseInstall,
+	dynamicPhases: []analysisrun.DynamicPhase{
+		analysisrun.DynamicPhaseInstall,
 	},
 }
 
 var cratesPkgManagerCombinedSandbox = PkgManager{
-	ecosystem:     api.EcosystemCratesIO,
+	ecosystem:     pkgecosystem.CratesIO,
 	image:         combinedDynamicAnalysisImage,
 	command:       "/usr/local/bin/analyze-rust.py",
 	latestVersion: getCratesLatest,
-	runPhases: []api.RunPhase{
-		api.RunPhaseInstall,
+	dynamicPhases: []analysisrun.DynamicPhase{
+		analysisrun.DynamicPhaseInstall,
 	},
 }

internal/pkgecosystem/download.go renamed to internal/pkgmanager/download.go

@@ -1,4 +1,4 @@
-package pkgecosystem
+package pkgmanager
 
 import (
 	"fmt"

internal/pkgecosystem/download_test.go renamed to internal/pkgmanager/download_test.go

@@ -1,10 +1,10 @@
-package pkgecosystem
+package pkgmanager
 
 import (
 	"os"
 	"testing"
 
-	"github.com/ossf/package-analysis/pkg/api"
+	"github.com/ossf/package-analysis/pkg/api/pkgecosystem"
 )
 
 type downloadTestSpec struct {
@@ -60,7 +60,7 @@ func TestDownloadNpmArchive(t *testing.T) {
 		{"pkgname=(invalid)", "fr(2t5j923)", "latest", true},
 	}
 
-	testDownload(t, tests, Manager(api.EcosystemNPM, false))
+	testDownload(t, tests, Manager(pkgecosystem.NPM, false))
 }
 
 func TestDownloadPyPIArchive(t *testing.T) {
@@ -70,7 +70,7 @@ func TestDownloadPyPIArchive(t *testing.T) {
 		{"pkgname=(invalid)", "fr(2t5j923)", "123", true},
 	}
 
-	testDownload(t, tests, Manager(api.EcosystemPyPI, false))
+	testDownload(t, tests, Manager(pkgecosystem.PyPI, false))
 }
 
 func TestDownloadToDirectory(t *testing.T) {