import "golang.org/x/pkgsite/internal/postgres"
Copyright 2019 The Go Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
details.go directory.go excluded.go experiment.go insert_module.go licenses.go package.go path.go postgres.go requeue.go search.go searchdoc.go test_helper.go version.go version_map.go versionstate.go
var (
// SearchLatencyDistribution aggregates search request latency by search
// query type.
SearchLatencyDistribution = &view.View{
Name: "go-discovery/search/latency",
Measure: searchLatency,
Aggregation: ochttp.DefaultLatencyDistribution,
Description: "Search latency, by result source query type.",
TagKeys: []tag.Key{keySearchSource},
}
// SearchResponseCount counts search responses by search query type.
SearchResponseCount = &view.View{
Name: "go-discovery/search/count",
Measure: searchLatency,
Aggregation: view.Count(),
Description: "Search count, by result source query type.",
TagKeys: []tag.Key{keySearchSource},
}
)GeneratePathTokens returns the subPaths and path token parts that will be indexed for search, which includes (1) the packagePath (2) all sub-paths of the packagePath (3) all parts for a path element that is delimited by a dash and (4) all parts of a path element that is delimited by a dot, except for the last element.
func GetFromSearchDocuments(ctx context.Context, t *testing.T, db *DB, packagePath string) (modulePath, version string, found bool)
GetFromSearchDocuments retrieves the module path and version for the given package path from the search_documents table. If the path is not in the table, the third return value is false.
InsertSampleDirectory tree inserts a set of packages for testing GetDirectory and frontend.FetchDirectoryDetails.
ResetTestDB truncates all data from the given test DB. It should be called after every test that mutates the database.
RunDBTests is a wrapper that runs the given testing suite in a test database named dbName. The given *DB reference will be set to the instantiated test database.
SearchDocumentSections computes the B and C sections of a Postgres search document from a package synopsis and a README. By "B section" and "C section" we mean the portion of the tsvector with weight "B" and "C", respectively.
The B section consists of the synopsis. The C section consists of the first sentence of the README. The D section consists of the remainder of the README. All sections are split into words and processed for replacements. Each section is limited to maxSectionWords words, and in addition the D section is limited to an initial fraction of the README, determined by maxReadmeFraction.
func UpsertSearchDocument(ctx context.Context, db *database.DB, args upsertSearchDocumentArgs) (err error)
UpsertSearchDocument inserts a row for each package in the module, if that package is the latest version and is not internal.
The given module should have already been validated via a call to validateModule.
UpsertSearchDocuments adds search information for mod ot the search_documents table.
type DB struct {
// contains filtered or unexported fields
}New returns a new postgres DB.
SetupTestDB creates a test database named dbName if it does not already exist, and migrates it to the latest schema from the migrations directory.
Close closes a DB.
DeleteModule deletes a Version from the database.
func (db *DB) DeleteOlderVersionFromSearchDocuments(ctx context.Context, modulePath, version string) (err error)
DeleteOlderVersionFromSearchDocuments deletes from search_documents every package with the given module path whose version is older than the given version. It is used when fetching a module with an alternative path. See internal/worker/fetch.go:fetchAndUpdateState.
func (db *DB) GetDirectory(ctx context.Context, path, modulePath, version string) (_ *internal.VersionedDirectory, err error)
GetDirectory returns a directory from the database, along with all of the data associated with that directory, including the package, imports, readme, documentation, and licenses.
GetExcludedPrefixes reads all the excluded prefixes from the database.
GetExperiments fetches all experiments in the database.
func (db *DB) GetImportedBy(ctx context.Context, pkgPath, modulePath string, limit int) (paths []string, err error)
GetImportedBy fetches and returns all of the packages that import the package with path. The returned error may be checked with derrors.IsInvalidArgument to determine if it resulted from an invalid package path or version.
Instead of supporting pagination, this query runs with a limit.
func (db *DB) GetImports(ctx context.Context, pkgPath, modulePath, version string) (paths []string, err error)
GetImports fetches and returns all of the imports for the package with pkgPath, modulePath and version.
The returned error may be checked with derrors.IsInvalidArgument to determine if it resulted from an invalid package path or version.
func (db *DB) GetLicenses(ctx context.Context, fullPath, modulePath, resolvedVersion string) (_ []*licenses.License, err error)
GetLicenses returns the licenses that applies to the fullPath for the given module version. It returns an InvalidArgument error if the module path or version is invalid.
func (db *DB) GetModuleInfo(ctx context.Context, modulePath, version string) (_ *internal.ModuleInfo, err error)
GetModuleInfo fetches a module version from the database with the primary key (module_path, version).
func (db *DB) GetModuleVersionState(ctx context.Context, modulePath, version string) (_ *internal.ModuleVersionState, err error)
GetModuleVersionState returns the current module version state for modulePath and version.
func (db *DB) GetNextModulesToFetch(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)
GetNextModulesToFetch returns the next batch of modules that need to be processed. We prioritize modules based on (1) whether it has status zero (never processed), (2) whether it is the latest version, (3) if it is an alternative module, and (4) the number of packages it has. We want to leave time-consuming modules until the end and process them at a slower rate to reduce database load and timeouts. We also want to leave alternative modules towards the end, since these will incur unnecessary deletes otherwise.
func (db *DB) GetPackageVersionState(ctx context.Context, pkgPath, modulePath, version string) (_ *internal.PackageVersionState, err error)
GetPackageVersionState returns the current package version state for pkgPath, modulePath and version.
func (db *DB) GetPackageVersionStatesForModule(ctx context.Context, modulePath, version string) (_ []*internal.PackageVersionState, err error)
GetPackageVersionStatesForModule returns the current package version states for modulePath and version.
func (db *DB) GetPackagesForSearchDocumentUpsert(ctx context.Context, before time.Time, limit int) (argsList []upsertSearchDocumentArgs, err error)
GetPackagesForSearchDocumentUpsert fetches search information for packages in search_documents whose update time is before the given time.
func (db *DB) GetPackagesInDirectory(ctx context.Context, dirPath, modulePath, resolvedVersion string) (_ []*internal.PackageMeta, err error)
GetPackagesInDirectory returns all of the packages in a directory from a module version, including the package that lives at dirPath, if present.
func (db *DB) GetPathInfo(ctx context.Context, path, inModulePath, inVersion string) (outModulePath, outVersion string, isPackage bool, err error)
GetPathInfo returns information about the "best" entity (module, path or directory) with the given path. The module and version arguments provide additional constraints. If the module is unknown, pass internal.UnknownModulePath; if the version is unknown, pass internal.LatestVersion.
The rules for picking the best are: 1. Match the module path and or version, if they are provided; 2. Prefer newer module versions to older, and release to pre-release; 3. In the unlikely event of two paths at the same version, pick the longer module path.
func (db *DB) GetRecentFailedVersions(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)
GetRecentFailedVersions returns versions that have most recently failed.
func (db *DB) GetRecentVersions(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)
GetRecentVersions returns recent versions that have been processed.
func (db *DB) GetStdlibPathsWithSuffix(ctx context.Context, suffix string) (paths []string, err error)
GetStdlibPathsWithSuffix returns information about all paths in the latest version of the standard library whose last component is suffix. A path that exactly match suffix is not included; the path must end with "/" + suffix.
We are only interested in actual standard library packages: not commands, which we happen to include in the stdlib module, and not directories (paths that do not contain a package).
func (db *DB) GetVersionMap(ctx context.Context, modulePath, requestedVersion string) (_ *internal.VersionMap, err error)
GetVersionMap fetches a version_map entry corresponding to the given modulePath and requestedVersion.
GetVersionStats queries the module_version_states table for aggregate information about the current state of module versions, grouping them by their current status code.
func (db *DB) GetVersionsForPath(ctx context.Context, path string) (_ []*internal.ModuleInfo, err error)
GetVersionsForPath returns a list of tagged versions sorted in descending semver order if any exist. If none, it returns the 10 most recent from a list of pseudo-versions sorted in descending semver order.
InsertExcludedPrefix inserts prefix into the excluded_prefixes table.
For real-time administration (e.g. DOS prevention), use the dbadmin tool. to exclude or unexclude a prefix. If the exclusion is permanent (e.g. a user request), also add the prefix and reason to the excluded.txt file.
InsertExperiment inserts a row into the experiments table.
func (db *DB) InsertIndexVersions(ctx context.Context, versions []*internal.IndexVersion) (err error)
InsertIndexVersions inserts new versions into the module_version_states table with a status of zero.
InsertModule inserts a version into the database using db.saveVersion, along with a search document corresponding to each of its packages.
IsExcluded reports whether the path matches the excluded list.
LatestIndexTimestamp returns the last timestamp successfully inserted into the module_version_states table.
func (db *DB) LegacyGetDirectory(ctx context.Context, dirPath, modulePath, version string, fields internal.FieldSet) (_ *internal.LegacyDirectory, err error)
LegacyGetDirectory returns the directory corresponding to the provided dirPath, modulePath, and version. The directory will contain all packages for that version, in sorted order by package path.
If version = internal.LatestVersion, the directory corresponding to the latest matching module version will be fetched.
fields is a set of fields to read (see internal.FieldSet and related definitions). If a field is not in fields, it will not be read from the DB and its value will be one of the XXXFieldMissing constants. Only certain large fields are treated specially in this way.
If more than one module ties for a given dirPath and version pair, and modulePath = internal.UnknownModulePath, the directory for the module with the longest module path will be fetched. For example, if there are two rows in the packages table: (1) path = "github.com/hashicorp/vault/api"
module_path = "github.com/hashicorp/vault"
AND (2) path = "github.com/hashicorp/vault/api"
module_path = "github.com/hashicorp/vault/api"
Only directories in the latter module will be returned.
Packages will be returned for a given dirPath if: (1) the package path has a prefix of dirPath (2) the dirPath has a prefix matching the package's module_path
For example, if the package "golang.org/x/tools/go/packages" in module "golang.org/x/tools" is in the database, it will match on: golang.org/x/tools golang.org/x/tools/go golang.org/x/tools/go/packages
It will not match on: golang.org/x/tools/g
func (db *DB) LegacyGetModuleInfo(ctx context.Context, modulePath string, version string) (_ *internal.LegacyModuleInfo, err error)
LegacyGetModuleInfo fetches a module version from the database with the primary key (module_path, version).
func (db *DB) LegacyGetModuleLicenses(ctx context.Context, modulePath, version string) (_ []*licenses.License, err error)
LegacyGetModuleLicenses returns all licenses associated with the given module path and version. These are the top-level licenses in the module zip file. It returns an InvalidArgument error if the module path or version is invalid.
func (db *DB) LegacyGetPackage(ctx context.Context, pkgPath, modulePath, version string) (_ *internal.LegacyVersionedPackage, err error)
LegacyGetPackage returns the a package from the database with the corresponding pkgPath, modulePath and version.
If version = internal.LatestVersion, the package corresponding to the latest matching module version will be fetched.
If more than one module tie for a given dirPath and version pair, and modulePath = internal.UnknownModulePath, the package in the module with the longest module path will be fetched. For example, if there are two rows in the packages table: (1) path = "github.com/hashicorp/vault/api"
module_path = "github.com/hashicorp/vault"
AND (2) path = "github.com/hashicorp/vault/api"
module_path = "github.com/hashicorp/vault/api"
The latter will be returned.
The returned error may be checked with errors.Is(err, derrors.InvalidArgument) to determine if it was caused by an invalid pkgPath, modulePath or version.
The returned error may be checked with errors.Is(err, derrors.InvalidArgument) to determine if it was caused by an invalid path or version.
func (db *DB) LegacyGetPackageLicenses(ctx context.Context, pkgPath, modulePath, version string) (_ []*licenses.License, err error)
LegacyGetPackageLicenses returns all licenses associated with the given package path and version. It returns an InvalidArgument error if the module path or version is invalid.
func (db *DB) LegacyGetPackagesInModule(ctx context.Context, modulePath, version string) (_ []*internal.LegacyPackage, err error)
LegacyGetPackagesInModule returns packages contained in the module version specified by modulePath and version. The returned packages will be sorted by their package path.
func (db *DB) LegacyGetPsuedoVersionsForModule(ctx context.Context, modulePath string) ([]*internal.ModuleInfo, error)
LegacyGetPsuedoVersionsForModule returns the 10 most recent from a list of pseudo-versions sorted in descending semver order.
func (db *DB) LegacyGetPsuedoVersionsForPackageSeries(ctx context.Context, pkgPath string) ([]*internal.ModuleInfo, error)
LegacyGetPsuedoVersionsForPackageSeries returns the 10 most recent from a list of pseudo-versions sorted in descending semver order. This list includes pseudo-versions of packages that have the same v1path.
func (db *DB) LegacyGetTaggedVersionsForModule(ctx context.Context, modulePath string) ([]*internal.ModuleInfo, error)
LegacyGetTaggedVersionsForModule returns a list of tagged versions sorted in descending semver order.
func (db *DB) LegacyGetTaggedVersionsForPackageSeries(ctx context.Context, pkgPath string) ([]*internal.ModuleInfo, error)
LegacyGetTaggedVersionsForPackageSeries returns a list of tagged versions sorted in descending semver order. This list includes tagged versions of packages that have the same v1path.
RemoveExperiment removes the specified experiment.
func (db *DB) Search(ctx context.Context, q string, limit, offset int) (_ []*internal.SearchResult, err error)
Search executes two search requests concurrently:
- a sequential scan of packages in descending order of popularity.
- all packages ("deep" search) using an inverted index to filter to search
terms.
The sequential scan takes significantly less time when searching for very common terms (e.g. "errors", "cloud", or "kubernetes"), due to its ability to exit early once the requested page of search results is provably complete.
Because 0 <= ts_rank() <= 1, we know that the highest score of any unscanned package is ln(e+N), where N is imported_by_count of the package we are currently considering. Therefore if the lowest scoring result of popular search is greater than ln(e+N), we know that we haven't missed any results and can return the search result immediately, cancelling other searches.
On the other hand, if the popular search is slow, it is likely that the search term is infrequent, and deep search will be fast due to our inverted gin index on search tokens.
The gap in this optimization is search terms that are very frequent, but rarely relevant: "int" or "package", for example. In these cases we'll pay the penalty of a deep search that scans nearly every package.
Underlying returns the *database.DB inside db.
UpdateExperiment updates the specified experiment with the provided rollout value.
func (db *DB) UpdateModuleVersionStatesForReprocessing(ctx context.Context, appVersion string) (err error)
UpdateModuleVersionStatesForReprocessing marks modules to be reprocessed that were processed prior to the provided appVersion.
func (db *DB) UpdateModuleVersionStatesWithStatus(ctx context.Context, status int, appVersion string) (err error)
func (db *DB) UpdateSearchDocumentsImportedByCount(ctx context.Context) (nUpdated int64, err error)
UpdateSearchDocumentsImportedByCount updates imported_by_count and imported_by_count_updated_at.
It does so by completely recalculating the imported-by counts from the imports_unique table.
UpdateSearchDocumentsImportedByCount returns the number of rows updated.
func (db *DB) UpsertModuleVersionState(ctx context.Context, modulePath, vers, appVersion string, timestamp time.Time, status int, goModPath string, fetchErr error, packageVersionStates []*internal.PackageVersionState) (err error)
UpsertModuleVersionState inserts or updates the module_version_state table with the results of a fetch operation for a given module version.
UpsertVersionMap inserts a version_map entry into the database.
type VersionStats struct {
LatestTimestamp time.Time
VersionCounts map[int]int // from status to number of rows
}VersionStats holds statistics extracted from the module_version_states table.
Package postgres imports 49 packages (graph) and is imported by 7 packages. Updated 2020-08-05. Refresh now. Tools for package owners.