import "golang.org/x/pkgsite/internal/postgres"
Package postgres provides functionality for reading and writing to the postgres database.
details.go excluded.go goldmark.go insert_module.go licenses.go path.go postgres.go requeue.go search.go searchdoc.go stdlib.go test_helper.go unit.go version.go version_map.go versionstate.go
var (
// SearchLatencyDistribution aggregates search request latency by search
// query type.
SearchLatencyDistribution = &view.View{
Name: "go-discovery/search/latency",
Measure: searchLatency,
Aggregation: ochttp.DefaultLatencyDistribution,
Description: "Search latency, by result source query type.",
TagKeys: []tag.Key{keySearchSource},
}
// SearchResponseCount counts search responses by search query type.
SearchResponseCount = &view.View{
Name: "go-discovery/search/count",
Measure: searchLatency,
Aggregation: view.Count(),
Description: "Search count, by result source query type.",
TagKeys: []tag.Key{keySearchSource},
}
)GeneratePathTokens returns the subPaths and path token parts that will be indexed for search, which includes (1) the packagePath (2) all sub-paths of the packagePath (3) all parts for a path element that is delimited by a dash and (4) all parts of a path element that is delimited by a dot, except for the last element.
func GetFromSearchDocuments(ctx context.Context, t *testing.T, db *DB, packagePath string) (modulePath, version string, found bool)
GetFromSearchDocuments retrieves the module path and version for the given package path from the search_documents table. If the path is not in the table, the third return value is false.
InsertSampleDirectory tree inserts a set of packages for testing GetUnit and frontend.FetchDirectoryDetails.
func NewHTMLRenderer(opts ...html.Option) renderer.NodeRenderer
NewHTMLRenderer creates a new HTMLRenderer for a readme.
ResetTestDB truncates all data from the given test DB. It should be called after every test that mutates the database.
RunDBTests is a wrapper that runs the given testing suite in a test database named dbName. The given *DB reference will be set to the instantiated test database.
SearchDocumentSections computes the B and C sections of a Postgres search document from a package synopsis and a README. By "B section" and "C section" we mean the portion of the tsvector with weight "B" and "C", respectively.
The B section consists of the synopsis. The C section consists of the first sentence of the README. The D section consists of the remainder of the README. All sections are split into words and processed for replacements. Each section is limited to maxSectionWords words, and in addition the D section is limited to an initial fraction of the README, determined by maxReadmeFraction.
type ASTTransformer struct{}ASTTransformer is a default transformer of the goldmark tree.
Transform transforms the given AST tree to remove an unnecessary child node from the image node. This is so that the summary generated doesn't the text content of an image block.
type DB struct {
// contains filtered or unexported fields
}New returns a new postgres DB.
NewBypassingLicenseCheck returns a new postgres DB that bypasses license checks. That means all data will be inserted and returned for non-redistributable modules, packages and directories.
SetupTestDB creates a test database named dbName if it does not already exist, and migrates it to the latest schema from the migrations directory.
Close closes a DB.
DeleteModule deletes a Version from the database.
func (db *DB) DeleteOlderVersionFromSearchDocuments(ctx context.Context, modulePath, resolvedVersion string) (err error)
DeleteOlderVersionFromSearchDocuments deletes from search_documents every package with the given module path whose version is older than the given version. It is used when fetching a module with an alternative path. See internal/worker/fetch.go:fetchAndUpdateState.
GetExcludedPrefixes reads all the excluded prefixes from the database.
func (db *DB) GetImportedBy(ctx context.Context, pkgPath, modulePath string, limit int) (paths []string, err error)
GetImportedBy fetches and returns all of the packages that import the package with path. The returned error may be checked with derrors.IsInvalidArgument to determine if it resulted from an invalid package path or version.
Instead of supporting pagination, this query runs with a limit.
func (db *DB) GetImportedByCount(ctx context.Context, pkgPath, modulePath string) (_ int, err error)
GetImportedByCount returns the number of packages that import pkgPath.
func (db *DB) GetLatestInfo(ctx context.Context, unitPath, modulePath string) (latest internal.LatestInfo, err error)
GetLatestInfo returns the latest information about the unit in the module. See internal.LatestInfo for documentation about the returned values.
func (db *DB) GetLatestMajorPathForV1Path(ctx context.Context, v1path string) (_ string, err error)
GetLatestMajorPathForV1Path reports the latest unit path in the series for the given v1path.
func (db *DB) GetModuleInfo(ctx context.Context, modulePath, resolvedVersion string) (_ *internal.ModuleInfo, err error)
GetModuleInfo fetches a module version from the database with the primary key (module_path, version).
func (db *DB) GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (_ *internal.Readme, err error)
GetModuleReadme returns the README corresponding to the modulePath and version.
func (db *DB) GetModuleVersionState(ctx context.Context, modulePath, resolvedVrsion string) (_ *internal.ModuleVersionState, err error)
GetModuleVersionState returns the current module version state for modulePath and version.
func (db *DB) GetNestedModules(ctx context.Context, modulePath string) (_ []*internal.ModuleInfo, err error)
GetNestedModules returns the latest major version of all nested modules given a modulePath path prefix with or without major version.
func (db *DB) GetNextModulesToFetch(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)
GetNextModulesToFetch returns the next batch of modules that need to be processed. We prioritize modules based on (1) whether it has status zero (never processed), (2) whether it is the latest version, (3) if it is an alternative module, and (4) the number of packages it has. We want to leave time-consuming modules until the end and process them at a slower rate to reduce database load and timeouts. We also want to leave alternative modules towards the end, since these will incur unnecessary deletes otherwise.
func (db *DB) GetPackageVersionState(ctx context.Context, pkgPath, modulePath, resolvedVersion string) (_ *internal.PackageVersionState, err error)
GetPackageVersionState returns the current package version state for pkgPath, modulePath and version.
func (db *DB) GetPackageVersionStatesForModule(ctx context.Context, modulePath, resolvedVersion string) (_ []*internal.PackageVersionState, err error)
GetPackageVersionStatesForModule returns the current package version states for modulePath and version.
func (db *DB) GetPackagesForSearchDocumentUpsert(ctx context.Context, before time.Time, limit int) (argsList []upsertSearchDocumentArgs, err error)
GetPackagesForSearchDocumentUpsert fetches search information for packages in search_documents whose update time is before the given time.
func (db *DB) GetRecentFailedVersions(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)
GetRecentFailedVersions returns versions that have most recently failed.
func (db *DB) GetRecentVersions(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error)
GetRecentVersions returns recent versions that have been processed.
func (db *DB) GetStdlibPathsWithSuffix(ctx context.Context, suffix string) (paths []string, err error)
GetStdlibPathsWithSuffix returns information about all paths in the latest version of the standard library whose last component is suffix. A path that exactly match suffix is not included; the path must end with "/" + suffix.
We are only interested in actual standard library packages: not commands, which we happen to include in the stdlib module, and not directories (paths that do not contain a package).
func (db *DB) GetUnit(ctx context.Context, um *internal.UnitMeta, fields internal.FieldSet) (_ *internal.Unit, err error)
GetUnit returns a unit from the database, along with all of the data associated with that unit.
func (db *DB) GetUnitMeta(ctx context.Context, fullPath, requestedModulePath, requestedVersion string) (_ *internal.UnitMeta, err error)
GetUnitMeta returns information about the "best" entity (module, path or directory) with the given path. The module and version arguments provide additional constraints. If the module is unknown, pass internal.UnknownModulePath; if the version is unknown, pass internal.LatestVersion.
The rules for picking the best are: 1. Match the module path and or version, if they are provided; 2. Prefer newer module versions to older, and release to pre-release; 3. In the unlikely event of two paths at the same version, pick the longer module path.
func (db *DB) GetVersionMap(ctx context.Context, modulePath, requestedVersion string) (_ *internal.VersionMap, err error)
GetVersionMap fetches a version_map entry corresponding to the given modulePath and requestedVersion.
func (db *DB) GetVersionMapsNon2xxStatus(ctx context.Context, paths []string, requestedVersion string) (_ []*internal.VersionMap, err error)
GetVersionMapsNon2xxStatus returns all of the version maps for the provided path and requested version if they are present.
GetVersionStats queries the module_version_states table for aggregate information about the current state of module versions, grouping them by their current status code.
func (db *DB) GetVersionsForPath(ctx context.Context, path string) (_ []*internal.ModuleInfo, err error)
GetVersionsForPath returns a list of tagged versions sorted in descending semver order if any exist. If none, it returns the 10 most recent from a list of pseudo-versions sorted in descending semver order.
InsertExcludedPrefix inserts prefix into the excluded_prefixes table.
For real-time administration (e.g. DOS prevention), use the dbadmin tool. to exclude or unexclude a prefix. If the exclusion is permanent (e.g. a user request), also add the prefix and reason to the excluded.txt file.
func (db *DB) InsertIndexVersions(ctx context.Context, versions []*internal.IndexVersion) (err error)
InsertIndexVersions inserts new versions into the module_version_states table with a status of zero.
InsertModule inserts a version into the database using db.saveVersion, along with a search document corresponding to each of its packages.
IsExcluded reports whether the path matches the excluded list. A path matches an entry on the excluded list if it equals the entry, or is a component-wise suffix of the entry. So path "bad/ness" matches entries "bad" and "bad/", but path "badness" matches neither of those.
LatestIndexTimestamp returns the last timestamp successfully inserted into the module_version_states table.
func (db *DB) Search(ctx context.Context, q string, limit, offset, maxResultCount int) (_ []*internal.SearchResult, err error)
Search executes two search requests concurrently:
- a sequential scan of packages in descending order of popularity.
- all packages ("deep" search) using an inverted index to filter to search
terms.
The sequential scan takes significantly less time when searching for very common terms (e.g. "errors", "cloud", or "kubernetes"), due to its ability to exit early once the requested page of search results is provably complete.
Because 0 <= ts_rank() <= 1, we know that the highest score of any unscanned package is ln(e+N), where N is imported_by_count of the package we are currently considering. Therefore if the lowest scoring result of popular search is greater than ln(e+N), we know that we haven't missed any results and can return the search result immediately, cancelling other searches.
On the other hand, if the popular search is slow, it is likely that the search term is infrequent, and deep search will be fast due to our inverted gin index on search tokens.
The gap in this optimization is search terms that are very frequent, but rarely relevant: "int" or "package", for example. In these cases we'll pay the penalty of a deep search that scans nearly every package.
StalenessTimestamp returns the index timestamp of the oldest module that is newer than the index timestamp of the youngest module we have processed. That is, let T be the maximum index timestamp of all processed modules. Then this function return the minimum index timestamp of unprocessed modules that is no less than T, or an error that wraps derrors.NotFound if there is none.
The name of the function is imprecise: there may be an older unprocessed module, if one newer than it has been processed.
We use this function to compute a metric that is a lower bound on the time it takes to process a module since it appeared in the index.
Underlying returns the *database.DB inside db.
func (db *DB) UpdateModuleVersionStatesForReprocessing(ctx context.Context, appVersion string) (err error)
UpdateModuleVersionStatesForReprocessing marks modules to be reprocessed that were processed prior to the provided appVersion.
func (db *DB) UpdateModuleVersionStatesWithStatus(ctx context.Context, status int, appVersion string) (err error)
func (db *DB) UpdateSearchDocumentsImportedByCount(ctx context.Context) (nUpdated int64, err error)
UpdateSearchDocumentsImportedByCount updates imported_by_count and imported_by_count_updated_at.
It does so by completely recalculating the imported-by counts from the imports_unique table.
UpdateSearchDocumentsImportedByCount returns the number of rows updated.
func (db *DB) UpsertModuleVersionState(ctx context.Context, modulePath, vers, appVersion string, timestamp time.Time, status int, goModPath string, fetchErr error, packageVersionStates []*internal.PackageVersionState) (err error)
UpsertModuleVersionState inserts or updates the module_version_state table with the results of a fetch operation for a given module version.
func (db *DB) UpsertSearchDocument(ctx context.Context, ddb *database.DB, args upsertSearchDocumentArgs) (err error)
UpsertSearchDocument inserts a row for each package in the module, if that package is the latest version and is not internal.
The given module should have already been validated via a call to validateModule.
UpsertVersionMap inserts a version_map entry into the database.
HTMLRenderer is a renderer.NodeRenderer implementation that renders pkg.go.dev readme features.
func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer)
RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs.
type VersionStats struct {
LatestTimestamp time.Time
VersionCounts map[int]int // from status to number of rows
}VersionStats holds statistics extracted from the module_version_states table.
Package postgres imports 56 packages (graph) and is imported by 7 packages. Updated 2021-01-15. Refresh now. Tools for package owners.