pkgsite: golang.org/x/pkgsite/internal/postgres Index | Files

package postgres

import "golang.org/x/pkgsite/internal/postgres"

Copyright 2019 The Go Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

Index

Package Files

details.go directory.go excluded.go experiment.go insert_module.go licenses.go package.go path.go postgres.go requeue.go search.go searchdoc.go test_helper.go version.go version_map.go versionstate.go

Variables

var (

    // SearchLatencyDistribution aggregates search request latency by search
    // query type.
    SearchLatencyDistribution = &view.View{
        Name:        "go-discovery/search/latency",
        Measure:     searchLatency,
        Aggregation: ochttp.DefaultLatencyDistribution,
        Description: "Search latency, by result source query type.",
        TagKeys:     []tag.Key{keySearchSource},
    }
    // SearchResponseCount counts search responses by search query type.
    SearchResponseCount = &view.View{
        Name:        "go-discovery/search/count",
        Measure:     searchLatency,
        Aggregation: view.Count(),
        Description: "Search count, by result source query type.",
        TagKeys:     []tag.Key{keySearchSource},
    }
)
var MemoryLoggingDisabled = true

func GeneratePathTokens Uses

func GeneratePathTokens(packagePath string) []string

GeneratePathTokens returns the subPaths and path token parts that will be indexed for search, which includes (1) the packagePath (2) all sub-paths of the packagePath (3) all parts for a path element that is delimited by a dash and (4) all parts of a path element that is delimited by a dot, except for the last element.

func GetFromSearchDocuments Uses

func GetFromSearchDocuments(ctx context.Context, t *testing.T, db *DB, packagePath string) (modulePath, version string, found bool)

GetFromSearchDocuments retrieves the module path and version for the given package path from the search_documents table. If the path is not in the table, the third return value is false.

func InsertSampleDirectoryTree Uses

func InsertSampleDirectoryTree(ctx context.Context, t *testing.T, testDB *DB)

InsertSampleDirectory tree inserts a set of packages for testing GetDirectory and frontend.FetchDirectoryDetails.

func ResetTestDB Uses

func ResetTestDB(db *DB, t *testing.T)

ResetTestDB truncates all data from the given test DB. It should be called after every test that mutates the database.

func RunDBTests Uses

func RunDBTests(dbName string, m *testing.M, testDB **DB)

RunDBTests is a wrapper that runs the given testing suite in a test database named dbName. The given *DB reference will be set to the instantiated test database.

func SearchDocumentSections Uses

func SearchDocumentSections(synopsis, readmeFilename, readme string) (b, c, d string)

SearchDocumentSections computes the B and C sections of a Postgres search document from a package synopsis and a README. By "B section" and "C section" we mean the portion of the tsvector with weight "B" and "C", respectively.

The B section consists of the synopsis. The C section consists of the first sentence of the README. The D section consists of the remainder of the README. All sections are split into words and processed for replacements. Each section is limited to maxSectionWords words, and in addition the D section is limited to an initial fraction of the README, determined by maxReadmeFraction.

func UpsertSearchDocument Uses

func UpsertSearchDocument(ctx context.Context, db *database.DB, args upsertSearchDocumentArgs) (err error)

UpsertSearchDocument inserts a row for each package in the module, if that package is the latest version and is not internal.

The given module should have already been validated via a call to validateModule.

func UpsertSearchDocuments Uses

func UpsertSearchDocuments(ctx context.Context, db *database.DB, mod *internal.Module) (err error)

UpsertSearchDocuments adds search information for mod ot the search_documents table.

type DB Uses

type DB struct {
    // contains filtered or unexported fields
}

func New Uses

func New(db *database.DB) *DB

New returns a new postgres DB.

func SetupTestDB Uses

func SetupTestDB(dbName string) (_ *DB, err error)

SetupTestDB creates a test database named dbName if it does not already exist, and migrates it to the latest schema from the migrations directory.

func (*DB) Close Uses

func (db *DB) Close() error

Close closes a DB.

func (*DB) DeleteModule Uses

func (db *DB) DeleteModule(ctx context.Context, modulePath, version string) (err error)

DeleteModule deletes a Version from the database.

func (*DB) DeleteOlderVersionFromSearchDocuments Uses

func (db *DB) DeleteOlderVersionFromSearchDocuments(ctx context.Context, modulePath, version string) (err error)

DeleteOlderVersionFromSearchDocuments deletes from search_documents every package with the given module path whose version is older than the given version. It is used when fetching a module with an alternative path. See internal/worker/fetch.go:fetchAndUpdateState.

func (*DB) GetDirectory Uses

func (db *DB) GetDirectory(ctx context.Context, path, modulePath, version string) (_ *internal.VersionedDirectory, err error)

GetDirectory returns a directory from the database, along with all of the data associated with that directory, including the package, imports, readme, documentation, and licenses.

func (*DB) GetExcludedPrefixes Uses

func (db *DB) GetExcludedPrefixes(ctx context.Context) ([]string, error)

GetExcludedPrefixes reads all the excluded prefixes from the database.

func (*DB) GetExperiments Uses

func (db *DB) GetExperiments(ctx context.Context) (_ []*internal.Experiment, err error)

GetExperiments fetches all experiments in the database.

func (*DB) GetImportedBy Uses

func (db *DB) GetImportedBy(ctx context.Context, pkgPath, modulePath string, limit int) (paths []string, err error)

GetImportedBy fetches and returns all of the packages that import the package with path. The returned error may be checked with derrors.IsInvalidArgument to determine if it resulted from an invalid package path or version.

Instead of supporting pagination, this query runs with a limit.

func (*DB) GetImports Uses

func (db *DB) GetImports(ctx context.Context, pkgPath, modulePath, version string) (paths []string, err error)

GetImports fetches and returns all of the imports for the package with pkgPath, modulePath and version.

The returned error may be checked with derrors.IsInvalidArgument to determine if it resulted from an invalid package path or version.

func (*DB) GetLicenses Uses

func (db *DB) GetLicenses(ctx context.Context, fullPath, modulePath, resolvedVersion string) (_ []*licenses.License, err error)

GetLicenses returns the licenses that applies to the fullPath for the given module version. It returns an InvalidArgument error if the module path or version is invalid.

func (*DB) GetModuleInfo Uses

func (db *DB) GetModuleInfo(ctx context.Context, modulePath, version string) (_ *internal.ModuleInfo, err error)

GetModuleInfo fetches a module version from the database with the primary key (module_path, version).

func (*DB) GetModuleVersionState Uses

func (db *DB) GetModuleVersionState(ctx context.Context, modulePath, version string) (_ *internal.ModuleVersionState, err error)

GetModuleVersionState returns the current module version state for modulePath and version.

func (*DB) GetNextModulesToFetch Uses

func (db *DB) GetNextModulesToFetch(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)

GetNextModulesToFetch returns the next batch of modules that need to be processed. We prioritize modules based on (1) whether it has status zero (never processed), (2) whether it is the latest version, (3) if it is an alternative module, and (4) the number of packages it has. We want to leave time-consuming modules until the end and process them at a slower rate to reduce database load and timeouts. We also want to leave alternative modules towards the end, since these will incur unnecessary deletes otherwise.

func (*DB) GetPackageVersionState Uses

func (db *DB) GetPackageVersionState(ctx context.Context, pkgPath, modulePath, version string) (_ *internal.PackageVersionState, err error)

GetPackageVersionState returns the current package version state for pkgPath, modulePath and version.

func (*DB) GetPackageVersionStatesForModule Uses

func (db *DB) GetPackageVersionStatesForModule(ctx context.Context, modulePath, version string) (_ []*internal.PackageVersionState, err error)

GetPackageVersionStatesForModule returns the current package version states for modulePath and version.

func (*DB) GetPackagesForSearchDocumentUpsert Uses

func (db *DB) GetPackagesForSearchDocumentUpsert(ctx context.Context, before time.Time, limit int) (argsList []upsertSearchDocumentArgs, err error)

GetPackagesForSearchDocumentUpsert fetches search information for packages in search_documents whose update time is before the given time.

func (*DB) GetPackagesInDirectory Uses

func (db *DB) GetPackagesInDirectory(ctx context.Context, dirPath, modulePath, resolvedVersion string) (_ []*internal.PackageMeta, err error)

GetPackagesInDirectory returns all of the packages in a directory from a module version, including the package that lives at dirPath, if present.

func (*DB) GetPathInfo Uses

func (db *DB) GetPathInfo(ctx context.Context, path, inModulePath, inVersion string) (outModulePath, outVersion string, isPackage bool, err error)

GetPathInfo returns information about the "best" entity (module, path or directory) with the given path. The module and version arguments provide additional constraints. If the module is unknown, pass internal.UnknownModulePath; if the version is unknown, pass internal.LatestVersion.

The rules for picking the best are: 1. Match the module path and or version, if they are provided; 2. Prefer newer module versions to older, and release to pre-release; 3. In the unlikely event of two paths at the same version, pick the longer module path.

func (*DB) GetRecentFailedVersions Uses

func (db *DB) GetRecentFailedVersions(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)

GetRecentFailedVersions returns versions that have most recently failed.

func (*DB) GetRecentVersions Uses

func (db *DB) GetRecentVersions(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)

GetRecentVersions returns recent versions that have been processed.

func (*DB) GetStdlibPathsWithSuffix Uses

func (db *DB) GetStdlibPathsWithSuffix(ctx context.Context, suffix string) (paths []string, err error)

GetStdlibPathsWithSuffix returns information about all paths in the latest version of the standard library whose last component is suffix. A path that exactly match suffix is not included; the path must end with "/" + suffix.

We are only interested in actual standard library packages: not commands, which we happen to include in the stdlib module, and not directories (paths that do not contain a package).

func (*DB) GetVersionMap Uses

func (db *DB) GetVersionMap(ctx context.Context, modulePath, requestedVersion string) (_ *internal.VersionMap, err error)

GetVersionMap fetches a version_map entry corresponding to the given modulePath and requestedVersion.

func (*DB) GetVersionStats Uses

func (db *DB) GetVersionStats(ctx context.Context) (_ *VersionStats, err error)

GetVersionStats queries the module_version_states table for aggregate information about the current state of module versions, grouping them by their current status code.

func (*DB) GetVersionsForPath Uses

func (db *DB) GetVersionsForPath(ctx context.Context, path string) (_ []*internal.ModuleInfo, err error)

GetVersionsForPath returns a list of tagged versions sorted in descending semver order if any exist. If none, it returns the 10 most recent from a list of pseudo-versions sorted in descending semver order.

func (*DB) InsertExcludedPrefix Uses

func (db *DB) InsertExcludedPrefix(ctx context.Context, prefix, user, reason string) (err error)

InsertExcludedPrefix inserts prefix into the excluded_prefixes table.

For real-time administration (e.g. DOS prevention), use the dbadmin tool. to exclude or unexclude a prefix. If the exclusion is permanent (e.g. a user request), also add the prefix and reason to the excluded.txt file.

func (*DB) InsertExperiment Uses

func (db *DB) InsertExperiment(ctx context.Context, e *internal.Experiment) (err error)

InsertExperiment inserts a row into the experiments table.

func (*DB) InsertIndexVersions Uses

func (db *DB) InsertIndexVersions(ctx context.Context, versions []*internal.IndexVersion) (err error)

InsertIndexVersions inserts new versions into the module_version_states table with a status of zero.

func (*DB) InsertModule Uses

func (db *DB) InsertModule(ctx context.Context, m *internal.Module) (err error)

InsertModule inserts a version into the database using db.saveVersion, along with a search document corresponding to each of its packages.

func (*DB) IsExcluded Uses

func (db *DB) IsExcluded(ctx context.Context, path string) (_ bool, err error)

IsExcluded reports whether the path matches the excluded list.

func (*DB) LatestIndexTimestamp Uses

func (db *DB) LatestIndexTimestamp(ctx context.Context) (_ time.Time, err error)

LatestIndexTimestamp returns the last timestamp successfully inserted into the module_version_states table.

func (*DB) LegacyGetDirectory Uses

func (db *DB) LegacyGetDirectory(ctx context.Context, dirPath, modulePath, version string, fields internal.FieldSet) (_ *internal.LegacyDirectory, err error)

LegacyGetDirectory returns the directory corresponding to the provided dirPath, modulePath, and version. The directory will contain all packages for that version, in sorted order by package path.

If version = internal.LatestVersion, the directory corresponding to the latest matching module version will be fetched.

fields is a set of fields to read (see internal.FieldSet and related definitions). If a field is not in fields, it will not be read from the DB and its value will be one of the XXXFieldMissing constants. Only certain large fields are treated specially in this way.

If more than one module ties for a given dirPath and version pair, and modulePath = internal.UnknownModulePath, the directory for the module with the longest module path will be fetched. For example, if there are two rows in the packages table: (1) path = "github.com/hashicorp/vault/api"

module_path = "github.com/hashicorp/vault"

AND (2) path = "github.com/hashicorp/vault/api"

module_path = "github.com/hashicorp/vault/api"

Only directories in the latter module will be returned.

Packages will be returned for a given dirPath if: (1) the package path has a prefix of dirPath (2) the dirPath has a prefix matching the package's module_path

For example, if the package "golang.org/x/tools/go/packages" in module "golang.org/x/tools" is in the database, it will match on: golang.org/x/tools golang.org/x/tools/go golang.org/x/tools/go/packages

It will not match on: golang.org/x/tools/g

func (*DB) LegacyGetModuleInfo Uses

func (db *DB) LegacyGetModuleInfo(ctx context.Context, modulePath string, version string) (_ *internal.LegacyModuleInfo, err error)

LegacyGetModuleInfo fetches a module version from the database with the primary key (module_path, version).

func (*DB) LegacyGetModuleLicenses Uses

func (db *DB) LegacyGetModuleLicenses(ctx context.Context, modulePath, version string) (_ []*licenses.License, err error)

LegacyGetModuleLicenses returns all licenses associated with the given module path and version. These are the top-level licenses in the module zip file. It returns an InvalidArgument error if the module path or version is invalid.

func (*DB) LegacyGetPackage Uses

func (db *DB) LegacyGetPackage(ctx context.Context, pkgPath, modulePath, version string) (_ *internal.LegacyVersionedPackage, err error)

LegacyGetPackage returns the a package from the database with the corresponding pkgPath, modulePath and version.

If version = internal.LatestVersion, the package corresponding to the latest matching module version will be fetched.

If more than one module tie for a given dirPath and version pair, and modulePath = internal.UnknownModulePath, the package in the module with the longest module path will be fetched. For example, if there are two rows in the packages table: (1) path = "github.com/hashicorp/vault/api"

module_path = "github.com/hashicorp/vault"

AND (2) path = "github.com/hashicorp/vault/api"

module_path = "github.com/hashicorp/vault/api"

The latter will be returned.

The returned error may be checked with errors.Is(err, derrors.InvalidArgument) to determine if it was caused by an invalid pkgPath, modulePath or version.

The returned error may be checked with errors.Is(err, derrors.InvalidArgument) to determine if it was caused by an invalid path or version.

func (*DB) LegacyGetPackageLicenses Uses

func (db *DB) LegacyGetPackageLicenses(ctx context.Context, pkgPath, modulePath, version string) (_ []*licenses.License, err error)

LegacyGetPackageLicenses returns all licenses associated with the given package path and version. It returns an InvalidArgument error if the module path or version is invalid.

func (*DB) LegacyGetPackagesInModule Uses

func (db *DB) LegacyGetPackagesInModule(ctx context.Context, modulePath, version string) (_ []*internal.LegacyPackage, err error)

LegacyGetPackagesInModule returns packages contained in the module version specified by modulePath and version. The returned packages will be sorted by their package path.

func (*DB) LegacyGetPsuedoVersionsForModule Uses

func (db *DB) LegacyGetPsuedoVersionsForModule(ctx context.Context, modulePath string) ([]*internal.ModuleInfo, error)

LegacyGetPsuedoVersionsForModule returns the 10 most recent from a list of pseudo-versions sorted in descending semver order.

func (*DB) LegacyGetPsuedoVersionsForPackageSeries Uses

func (db *DB) LegacyGetPsuedoVersionsForPackageSeries(ctx context.Context, pkgPath string) ([]*internal.ModuleInfo, error)

LegacyGetPsuedoVersionsForPackageSeries returns the 10 most recent from a list of pseudo-versions sorted in descending semver order. This list includes pseudo-versions of packages that have the same v1path.

func (*DB) LegacyGetTaggedVersionsForModule Uses

func (db *DB) LegacyGetTaggedVersionsForModule(ctx context.Context, modulePath string) ([]*internal.ModuleInfo, error)

LegacyGetTaggedVersionsForModule returns a list of tagged versions sorted in descending semver order.

func (*DB) LegacyGetTaggedVersionsForPackageSeries Uses

func (db *DB) LegacyGetTaggedVersionsForPackageSeries(ctx context.Context, pkgPath string) ([]*internal.ModuleInfo, error)

LegacyGetTaggedVersionsForPackageSeries returns a list of tagged versions sorted in descending semver order. This list includes tagged versions of packages that have the same v1path.

func (*DB) RemoveExperiment Uses

func (db *DB) RemoveExperiment(ctx context.Context, name string) (err error)

RemoveExperiment removes the specified experiment.

func (*DB) Search Uses

func (db *DB) Search(ctx context.Context, q string, limit, offset int) (_ []*internal.SearchResult, err error)

Search executes two search requests concurrently:

- a sequential scan of packages in descending order of popularity.
- all packages ("deep" search) using an inverted index to filter to search
  terms.

The sequential scan takes significantly less time when searching for very common terms (e.g. "errors", "cloud", or "kubernetes"), due to its ability to exit early once the requested page of search results is provably complete.

Because 0 <= ts_rank() <= 1, we know that the highest score of any unscanned package is ln(e+N), where N is imported_by_count of the package we are currently considering. Therefore if the lowest scoring result of popular search is greater than ln(e+N), we know that we haven't missed any results and can return the search result immediately, cancelling other searches.

On the other hand, if the popular search is slow, it is likely that the search term is infrequent, and deep search will be fast due to our inverted gin index on search tokens.

The gap in this optimization is search terms that are very frequent, but rarely relevant: "int" or "package", for example. In these cases we'll pay the penalty of a deep search that scans nearly every package.

func (*DB) Underlying Uses

func (db *DB) Underlying() *database.DB

Underlying returns the *database.DB inside db.

func (*DB) UpdateExperiment Uses

func (db *DB) UpdateExperiment(ctx context.Context, e *internal.Experiment) (err error)

UpdateExperiment updates the specified experiment with the provided rollout value.

func (*DB) UpdateModuleVersionStatesForReprocessing Uses

func (db *DB) UpdateModuleVersionStatesForReprocessing(ctx context.Context, appVersion string) (err error)

UpdateModuleVersionStatesForReprocessing marks modules to be reprocessed that were processed prior to the provided appVersion.

func (*DB) UpdateModuleVersionStatesWithStatus Uses

func (db *DB) UpdateModuleVersionStatesWithStatus(ctx context.Context, status int, appVersion string) (err error)

func (*DB) UpdateSearchDocumentsImportedByCount Uses

func (db *DB) UpdateSearchDocumentsImportedByCount(ctx context.Context) (nUpdated int64, err error)

UpdateSearchDocumentsImportedByCount updates imported_by_count and imported_by_count_updated_at.

It does so by completely recalculating the imported-by counts from the imports_unique table.

UpdateSearchDocumentsImportedByCount returns the number of rows updated.

func (*DB) UpsertModuleVersionState Uses

func (db *DB) UpsertModuleVersionState(ctx context.Context, modulePath, vers, appVersion string, timestamp time.Time, status int, goModPath string, fetchErr error, packageVersionStates []*internal.PackageVersionState) (err error)

UpsertModuleVersionState inserts or updates the module_version_state table with the results of a fetch operation for a given module version.

func (*DB) UpsertVersionMap Uses

func (db *DB) UpsertVersionMap(ctx context.Context, vm *internal.VersionMap) (err error)

UpsertVersionMap inserts a version_map entry into the database.

type VersionStats Uses

type VersionStats struct {
    LatestTimestamp time.Time
    VersionCounts   map[int]int // from status to number of rows
}

VersionStats holds statistics extracted from the module_version_states table.

Package postgres imports 49 packages (graph) and is imported by 7 packages. Updated 2020-08-05. Refresh now. Tools for package owners.