Documentation ¶
Overview ¶
Example (CheckExistingQuants) ¶
This Example demonstrates how to check for existing published quants.
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() datasetsbucket := "datasets-bucket" mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/publications.json"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(` { "Datasets": [ { "dataset-id": "mydataset", "job-id": "blah", "publications": [ { "publisher": "abc", "version": 1, "timestamp": "2018-09-21T12:42:31Z" }, { "publisher": "abc", "version": 2, "timestamp": "2018-09-22T12:42:31Z" } ] } ] } `))), }, } fs := fileaccess.MakeS3Access(&mockS3) // Check for the latest version of the published quant. It should be version 2. i, err := checkCurrentlyPublishedQuantVersion(fs, datasetsbucket, "mydataset") if err != nil { fmt.Printf("%v\n", err) } fmt.Printf("Version found: %v\n", i)
Output: Version found: 2
Example (CheckExistingQuantsUnsorted) ¶
This Example we demonstrate sort capabilities for unordered publication times.
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() datasetsbucket := "datasets-bucket" mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/publications.json"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(` { "Datasets": [ { "dataset-id": "mydataset", "job-id": "blah", "publications": [ { "publisher": "abc", "version": 2, "timestamp": "2018-09-22T12:42:31Z" }, { "publisher": "abc", "version": 1, "timestamp": "2018-09-21T12:42:31Z" } ] } ] } `))), }, } fs := fileaccess.MakeS3Access(&mockS3) // Check for the latest version of the published quant. It should be version 2. But we have deliberately switched // the order of the publications to ensure it doesn't match the last in the list. i, err := checkCurrentlyPublishedQuantVersion(fs, datasetsbucket, "mydataset") if err != nil { fmt.Printf("%v\n", err) } fmt.Printf("Version found: %v\n", i)
Output: Version found: 2
Example (CleanLogName) ¶
// Don't fix it... fmt.Println(cleanLogName("node00001_data.log")) // Do fix it... fmt.Println(cleanLogName("node00001.pmcs_stdout.log")) // Do fix it... fmt.Println(cleanLogName("NODE00001.PMCS_stdout.log"))
Output: node00001_data.log node00001_stdout.log NODE00001_stdout.log
Example (CombineQuantOutputs_BadPMC) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, RTT 30, 5.1, 400, 7890 12, 6.1, 405, 7800 `))), }, { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row2 PMC, CaO_%, CaO_int, RTT NaN, 7.1, 415, 7840 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"}) fmt.Printf("%v\n", err) fmt.Println(combinedCSV)
Output: Failed to combine map segment: JobData/abc123/output/node002.pmcs_result.csv, invalid PMC NaN at line 3
Example (CombineQuantOutputs_DownloadError) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, RTT 30, 5.1, 400, 7890 12, 6.1, 405, 7800 `))), }, nil, } fs := fileaccess.MakeS3Access(&mockS3) combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"}) fmt.Printf("%v\n", err) fmt.Println(combinedCSV)
Output: Failed to combine map segment: JobData/abc123/output/node002.pmcs_result.csv
Example (CombineQuantOutputs_DuplicatePMC) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"), }, { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node003.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, RTT 30, 5.1, 400, 7890 12, 6.1, 405, 7800 `))), }, { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row2 PMC, CaO_%, CaO_int, RTT 18, 7.1, 415, 7840 `))), }, { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row3 PMC, CaO_%, CaO_int, RTT 3, 1.1, 450, 7830 30, 1.3, 451, 7833 40, 8.1, 455, 7870 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"}) fmt.Printf("%v\n", err) fmt.Println(combinedCSV)
Output: <nil> The custom header PMC, CaO_%, CaO_int, RTT 3, 1.1, 450, 7830 12, 6.1, 405, 7800 18, 7.1, 415, 7840 30, 5.1, 400, 7890 30, 1.3, 451, 7833 40, 8.1, 455, 7870
Example (CombineQuantOutputs_LastLineCutOff) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, RTT 30, 5.1, 400, 7890 12, 6.1, 405, 7800 `))), }, { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row2 PMC, CaO_%, CaO_int, RTT 31 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"}) fmt.Printf("%v\n", err) fmt.Println(combinedCSV)
Output: Failed to combine map segment: JobData/abc123/output/node002.pmcs_result.csv, no PMC at line 3
Example (CombineQuantOutputs_OK) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"), }, { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node003.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, RTT 30, 5.1, 400, 7890 12, 6.1, 405, 7800 `))), }, { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row2 PMC, CaO_%, CaO_int, RTT 18, 7.1, 415, 7840 `))), }, { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row3 PMC, CaO_%, CaO_int, RTT 3, 1.1, 450, 7830 40, 8.1, 455, 7870 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"}) fmt.Printf("%v\n", err) fmt.Println(combinedCSV)
Output: <nil> The custom header PMC, CaO_%, CaO_int, RTT 3, 1.1, 450, 7830 12, 6.1, 405, 7800 18, 7.1, 415, 7840 30, 5.1, 400, 7890 40, 8.1, 455, 7870
Example (ConvertQuantificationData) ¶
data := csvData{ []string{"PMC", "Ca_%", "Ca_int", "SCLK", "Ti_%", "filename", "RTT"}, [][]string{ []string{"23", "1.5", "5", "11111", "4", "fileA.msa", "44"}, []string{"70", "3.4", "32", "12345", "4.21", "fileB.msa", "45"}, }, } result, err := convertQuantificationData(data, []string{"PMC", "RTT", "SCLK", "filename"}) fmt.Printf("%v|%v\n", result, err)
Output: {[Ca_% Ca_int Ti_%] [F I F] [{23 44 11111 fileA.msa [1.5 5 4]} {70 45 12345 fileB.msa [3.4 32 4.21]}]}|<nil>
Example (DecodeMapFileNameColumn) ¶
rt, det, err := decodeMapFileNameColumn("file.txt") fmt.Printf("%v|%v|%v\n", rt, det, err) rt, det, err = decodeMapFileNameColumn("Normal_A") fmt.Printf("%v|%v|%v\n", rt, det, err) rt, det, err = decodeMapFileNameColumn("Normal_A_MyRoiID") fmt.Printf("%v|%v|%v\n", rt, det, err) rt, det, err = decodeMapFileNameColumn("Dwell_B") fmt.Printf("%v|%v|%v\n", rt, det, err) rt, det, err = decodeMapFileNameColumn("Normal_C") fmt.Printf("%v|%v|%v\n", rt, det, err) rt, det, err = decodeMapFileNameColumn("LongRead_B") fmt.Printf("%v|%v|%v\n", rt, det, err) rt, det, err = decodeMapFileNameColumn("Scotland_something_00012.msa") fmt.Printf("%v|%v|%v\n", rt, det, err) rt, det, err = decodeMapFileNameColumn("Scotland_something_00012_10keV_33.msa") fmt.Printf("%v|%v|%v\n", rt, det, err) rt, det, err = decodeMapFileNameColumn("Normal_A_0123456789_873495_455.msa") fmt.Printf("%v|%v|%v\n", rt, det, err)
Output: ||decodeMapFileNameColumn: Invalid READTYPE in filename: "file.txt" Normal|A|<nil> Normal|A|<nil> Dwell|B|<nil> ||decodeMapFileNameColumn: Invalid DETECTOR_ID in filename: "Normal_C" ||decodeMapFileNameColumn: Invalid READTYPE in filename: "LongRead_B" ||decodeMapFileNameColumn: Invalid READTYPE in filename: "Scotland_something_00012.msa" ||decodeMapFileNameColumn: Invalid READTYPE in filename: "Scotland_something_00012_10keV_33.msa" Normal|A|<nil>
Example (EstimateNodeCount) ¶
5x11, 4035 PMCs, 9 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:23 (with params)
=> 4035*2=8070 spectra on 160 cores in 203 sec => 50.44 spectra/core in 203 sec => 4.02sec/spectra
5x11, 4035 PMCs, 6 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:47 (with params)
=> 4035*2=8070 spectra on 160 cores in 167 sec => 50.44 spectra/core in 167 sec => 3.31sec/spectra
5x11, 4035 PMCs, 3 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 1:52 (with params)
=> 4035*2=8070 spectra on 160 cores in 112 sec => 50.44 spectra/core in 112 sec => 2.22sec/spectra
5x11, 4035 PMCs, 3 elements, 8 cores (7.5 allocation in kubernetes), 10 nodes. Runtime: 3:32 (with params)
=> 4035*2=8070 spectra on 80 cores in 212 sec => 100.88 spectra/core in 212 sec => 2.10 sec/spectra
3 elements => 2.10sec/spectra 3 elements => 2.22sec/spectra
3 elems jumped 1.09sec
6 elements => 3.31sec/spectra
3 elems jumped 0.71sec
9 elements => 4.02sec/spectra
Assumptions:
Lets make this calcatable: 9elem=4sec/spectra, 3elem = 2sec/spectra, linearly interpolate in this range
Works out to elements = 3*sec - 3
To calculate node count, we are given Core count, Runtime desired, Spectra count, Element count
Using the above: Runtime = Spectra*SpectraRuntime / (Core*Nodes) Nodes = Spectra*SpectraRuntime / (Runtime * Core)
SpectraRuntime is calculated using the above formula: Elements = 3 * Sec - 3 SpectraRuntime = (Elements+3) / 3
Nodes = Spectra*((Elements + 3) / 3) / (RuntimeDesired * Cores) Nodes = Spectra*(Elements+3) / 3*(RuntimeDesired * Cores)
Example using the values from above: Nodes = 8070*(3+3)/(3*120*8) Nodes = 8070*6/5088 = 9.5, close to 10
Nodes = 8070*(9+3)/(3*203*8) Nodes = 96840 / 4872 = 19.9, close to 20
Nodes = 8070*(6+3)/(3*167*8) Nodes = 72630 / 4008 = 18.12, close to 20
If we're happy to run 6 elems, 8070 spectra, 8 cores in 5 minutes: Nodes = 8070*(6+3) / (3*300*8) Nodes = 72630 / 7200 = 10 nodes... seems reasonable
// Based on experimental runs in: https://github.com/pixlise/core/-/issues/113 // Can only use the ones where we had allcoation of 7.5 set in kubernetes, because the others weren't maxing out cores // 5x11, 4035 PMCs, 3 elements, 8 cores (7.5 allocation in kubernetes), 10 nodes. Runtime: 3:22 fmt.Println(estimateNodeCount(4035*2, 3, 3*60+22, 8, 50)) // 5x11, 4035 PMCs, 3 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 1:52 (with params) fmt.Println(estimateNodeCount(4035*2, 3, 60+52, 8, 50)) // 5x11, 4035 PMCs, 4 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:11 (with params) fmt.Println(estimateNodeCount(4035*2, 4, 2*60+11, 8, 50)) // 5x11, 4035 PMCs, 5 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:26 (with params) fmt.Println(estimateNodeCount(4035*2, 5, 2*60+26, 8, 50)) // 5x11, 4035 PMCs, 6 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:47 (with params) fmt.Println(estimateNodeCount(4035*2, 6, 2*60+47, 8, 50)) // 5x11, 4035 PMCs, 7 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:55 (no params though) fmt.Println(estimateNodeCount(4035*2, 7, 2*60+55, 8, 50)) // 5x11, 4035 PMCs, 8 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:12 (with params) fmt.Println(estimateNodeCount(4035*2, 8, 3*60+12, 8, 50)) // 5x11, 4035 PMCs, 9 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:23 (with params) fmt.Println(estimateNodeCount(4035*2, 9, 3*60+23, 8, 50)) // 5x11, 4035 PMCs, 10 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:35 (with params) fmt.Println(estimateNodeCount(4035*2, 10, 3*60+35, 8, 50)) // 5x11, 4035 PMCs, 11 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:46 (with params) fmt.Println(estimateNodeCount(4035*2, 11, 3*60+46, 8, 50)) // 5x5, 1769 PMCs, 11 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 1:47 (with params) fmt.Println(estimateNodeCount(1769*2, 11, 60+47, 8, 50)) // 5x5, 1769 PMCs, 4 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 0:59 (with params) fmt.Println(estimateNodeCount(1769*2, 4, 59, 8, 50)) // Ensure the max cores have an effect fmt.Println(estimateNodeCount(1769*2, 4, 59, 8, 6)) // It's a bit unfortunate we ran all but 1 tests on the same number of cores, but // the above data varies the spectra count, element count and expected runtime // Below we'd expect 20 for all answers except the first one, but there's a slight (and // allowable) drift because we're not exactly spot on with our estimate, and there's // fixed overhead time we aren't even calculating properly
Output: 10 18 18 18 18 19 19 20 20 21 19 17 6
Example (FilesPerNode) ¶
fmt.Println(filesPerNode(8088, 5)) fmt.Println(filesPerNode(8068, 3))
Output: 1619 2690
Example (FilterListItems) ¶
// Should just filter indexes that are valid idxToIgnoreMap := map[int]bool{ -9: true, 1: true, 2: true, 5: true, 6: true, } fmt.Println(filterListItems([]string{"snowboarding", "is", "awesome", "says", "Peter", "Nemere"}, idxToIgnoreMap))
Output: [snowboarding says Peter]
Example (GenerateMetFiles) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() datasetsbucket := "datasets-bucket" mockS3.ExpPutObjectInput = []s3.PutObjectInput{ { Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/Staging/mydatasetid/myjobid.csv.met"), Body: bytes.NewReader([]byte(`{ "description": "PiQuant Results File" }`)), }, { Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/Staging/mydatasetid/summary-myjobid.json.met"), Body: bytes.NewReader([]byte(`{ "description": "PiQuant Runtime Parameters File" }`)), }, } mockS3.QueuedPutObjectOutput = []*s3.PutObjectOutput{ {}, {}, } datasetid := "mydatasetid" jobid := "myjobid" products := ProductSet{ OcsPath: "", SourceBucket: "", SourcePrefix: "", DatasetID: datasetid, JobID: jobid, PqrFileName: "myjobid.csv", PqrMetaFileName: "myjobid.csv.met", PqpFileName: "summary-myjobid.json", PqpMetaFileName: "summary-myjobid.json.met", } fs := fileaccess.MakeS3Access(&mockS3) err := stageMetFiles(fs, datasetsbucket, products) if err != nil { fmt.Printf("%v", err) }
Output:
Example (GetElements) ¶
fmt.Printf("%v", getElements([]string{"PMC", "SCLK", "Ca_%", "Ti_%", "Ca_int", "Ti_int", "livetime", "Mg_%", "chisq"}))
Output: [Ca Ti Mg]
Example (GetInterestingColIndexes) ¶
header := []string{"PMC", "K_%", "Ca_%", "Fe_%", "K_int", "Ca_int", "Fe_int", "K_err", "Ca_err", "Fe_err", "total_counts", "livetime", "chisq", "eVstart", "eV/ch", "res", "iter", "filename", "Events", "Triggers", "SCLK", "RTT"} interesting, err := getInterestingColIndexes(header, []string{"PMC", "filename", "SCLK", "RTT"}) fmt.Printf("\"%v\" \"%v\"\n", interesting, err) interesting, err = getInterestingColIndexes(header, []string{"K_%", "total_counts"}) fmt.Printf("\"%v\" \"%v\"\n", interesting, err) // Bad cases interesting, err = getInterestingColIndexes(header, []string{"PMC", "TheFileName", "SCLK", "RTT"}) fmt.Printf("\"%v\" \"%v\"\n", interesting, err) header[5] = "SCLK" interesting, err = getInterestingColIndexes(header, []string{"PMC", "TheFileName", "SCLK", "RTT"}) fmt.Printf("\"%v\" \"%v\"\n", interesting, err) // 22 header items...
Output: "map[PMC:0 RTT:21 SCLK:20 filename:17]" "<nil>" "map[K_%:1 total_counts:10]" "<nil>" "map[]" "CSV column missing: TheFileName" "map[]" "Duplicate CSV column: SCLK"
Example (MakeColumnTypeList) ¶
data := csvData{[]string{"a", "b", "c", "d", "e"}, [][]string{[]string{"1.11111", "2", "3.1415962", "5", "6"}}} result, err := makeColumnTypeList(data, map[int]bool{2: true, 3: true}) fmt.Printf("%v|%v\n", result, err) result, err = makeColumnTypeList(data, map[int]bool{}) fmt.Printf("%v|%v\n", result, err) // Bad type data = csvData{[]string{"a", "b", "c", "d", "e"}, [][]string{[]string{"1.11111", "Wanaka", "3.1415962", "5"}}} result, err = makeColumnTypeList(data, map[int]bool{2: true, 3: true}) fmt.Printf("%v|%v\n", result, err) // Skipping the string 1 should make it work... result, err = makeColumnTypeList(data, map[int]bool{1: true, 3: true}) fmt.Printf("%v|%v\n", result, err)
Output: [F I I]|<nil> [F I F I I]|<nil> [F]|Failed to parse "Wanaka" as float or int at col 1/row 0 [F F]|<nil>
Example (MakeIndividualPMCListFileContents_AB) ¶
fmt.Println(makeIndividualPMCListFileContents([]int32{15, 7, 388}, "5x11dataset.bin", false, false, map[int32]bool{}))
Output: 5x11dataset.bin 15|Normal|A 15|Normal|B 7|Normal|A 7|Normal|B 388|Normal|A 388|Normal|B <nil>
Example (MakeIndividualPMCListFileContents_AB_Dwell) ¶
fmt.Println(makeIndividualPMCListFileContents([]int32{15, 7, 388}, "5x11dataset.bin", false, true, map[int32]bool{15: true}))
Output: 5x11dataset.bin 15|Normal|A,15|Dwell|A 15|Normal|B,15|Dwell|B 7|Normal|A 7|Normal|B 388|Normal|A 388|Normal|B <nil>
Example (MakeIndividualPMCListFileContents_Combined) ¶
fmt.Println(makeIndividualPMCListFileContents([]int32{15, 7, 388}, "5x11dataset.bin", true, false, map[int32]bool{}))
Output: 5x11dataset.bin 15|Normal|A,15|Normal|B 7|Normal|A,7|Normal|B 388|Normal|A,388|Normal|B <nil>
Example (MakeIndividualPMCListFileContents_Combined_Dwell) ¶
fmt.Println(makeIndividualPMCListFileContents([]int32{15, 7, 388}, "5x11dataset.bin", true, true, map[int32]bool{15: true}))
Output: 5x11dataset.bin 15|Normal|A,15|Normal|B,15|Dwell|A,15|Dwell|B 7|Normal|A,7|Normal|B 388|Normal|A,388|Normal|B <nil>
Example (MakeJobStartingParametersWithPMCCount) ¶
Making sure our embedded structure copying works
in := JobStartingParametersWithPMCs{ []int32{4, 59, 444, 2313, 329}, &JobStartingParameters{ Name: "name", DataBucket: "databucket", //DatasetsBucket: "datasetsbucket", //ConfigBucket: "configbucket", DatasetPath: "datasetpath", DatasetID: "datasetid", PiquantJobsBucket: "jobbucket", DetectorConfig: "config", Elements: []string{"Ti", "Al", "Ca"}, Parameters: "params", RunTimeSec: 39, CoresPerNode: 3, StartUnixTime: 33332222, Creator: pixlUser.UserInfo{ Name: "creator", UserID: "creator-id-123", Email: "niko@rockstar.com", Permissions: map[string]bool{"read:something": true, "read:another": true}, }, RoiID: "roiID", ElementSetID: "elemSetID", PIQUANTVersion: "3.0.3", Command: "map", }, } out := MakeJobStartingParametersWithPMCCount(in) fmt.Printf("pmc=%v, name=%v, elements=%v", out.PMCCount, out.Name, out.Elements)
Output: pmc=5, name=name, elements=[Ti Al Ca]
Example (MakeQuantJobPMCLists) ¶
fmt.Println(makeQuantJobPMCLists([]int32{1, 2, 3, 4, 5, 6, 7, 8}, 3)) fmt.Println(makeQuantJobPMCLists([]int32{1, 2, 3, 4, 5, 6, 7, 8, 9}, 3)) fmt.Println(makeQuantJobPMCLists([]int32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 3))
Output: [[1 2 3] [4 5 6] [7 8]] [[1 2 3] [4 5 6] [7 8 9]] [[1 2 3] [4 5 6] [7 8 9] [10]]
Example (MakeQuantProducts) ¶
Test code to ensure we make quant products correctly.
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() datasetsbucket := "datasets-bucket" usersbucket := "users-bucket" mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(usersbucket), Key: aws.String("UserContent/shared/mydatasetid/Quantifications/summary-myjobid.json"), }, { Bucket: aws.String(datasetsbucket), Key: aws.String("Datasets/mydatasetid/summary.json"), }, } // Real quant summary and dataset data from S3 mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(` { "shared": false, "params": { "pmcsCount": 2809, "name": "sum then quantify", "dataBucket": "prodstack-persistencepixlisedata4f446ecf-m36oehuca7uc", "datasetPath": "Datasets/069927431/dataset.bin", "datasetID": "069927431", "jobBucket": "prodstack-persistencepiquantjobs65c7175e-12qccz2o7aimo", "detectorConfig": "PIXL/PiquantConfigs/v6", "elements": [ "Ca", "Ti", "Fe", "Al" ], "parameters": "", "runTimeSec": 60, "coresPerNode": 4, "startUnixTime": 1630629236, "creator": { "name": "peternemere", "user_id": "5de45d85ca40070f421a3a34", "email": "peternemere@gmail.com" }, "roiID": "", "elementSetID": "", "piquantVersion": "registry.github.com/pixlise/piquant/runner:3.2.8-ALPHA", "quantMode": "CombinedBulk", "comments": "", "roiIDs": [ "scxb9y7xk027mcqq", "ee4e94n3fsu3lcqq", "4vvypcnk1xtd4sn2", "w28i1d4imca1o24e" ] }, "elements": [ "CaO", "TiO2", "FeO-T", "Al2O3" ], "jobId": "x80xad4l7k6vzalf", "status": "complete", "message": "Nodes ran: 1", "endUnixTime": 1630629323, "outputFilePath": "UserContent/5de45d85ca40070f421a3a34/069927431/Quantifications", "piquantLogList": [ "node00001_piquant.log", "node00001_stdout.log" ] } `))), }, { Body: ioutil.NopCloser(bytes.NewReader([]byte(` { "dataset_id": "069927431", "group": "PIXL-FM", "drive_id": 0, "site_id": 7, "target_id": "?", "site": "", "target": "", "title": "Bellegarde", "sol": "0187", "rtt": 69927431, "sclk": 683483744, "context_image": "PCW_0187_0683484439_000RCM_N00700000699274310005075J04.png", "location_count": 2816, "data_file_size": 17623427, "context_images": 21, "normal_spectra": 5618, "dwell_spectra": 20, "bulk_spectra": 2, "max_spectra": 2, "pseudo_intensities": 2809, "detector_config": "PIXL", "create_unixtime_sec": 1642519029 } `))), }, } datasetid := "mydatasetid" jobid := "myjobid" fs := fileaccess.MakeS3Access(&mockS3) ocsproducts, err := makeQuantProducts(fs, usersbucket, datasetsbucket, datasetid, jobid, 1) if err != nil { fmt.Printf("%v", err) } ocsjson, err := json.MarshalIndent(ocsproducts, "", " ") if err != nil { fmt.Printf("%v", err) } fmt.Printf("%v", string(ocsjson))
Output: { "OcsPath": "/ods/surface/sol/00187/soas/rdr/pixl/PQA", "SourceBucket": "users-bucket", "SourcePrefix": "Publish/Staging", "DatasetID": "mydatasetid", "JobID": "myjobid", "PqrFileName": "PES_0187_0683484439_000PQR_N00700000699274310005075J01.CSV", "PqrMetaFileName": "PES_0187_0683484439_000PQR_N00700000699274310005075J01.CSV.MET", "PqpFileName": "PES_0187_0683484439_000PQP_N00700000699274310005075J01.JSON", "PqpMetaFileName": "PES_0187_0683484439_000PQP_N00700000699274310005075J01.JSON.MET" }
Example (MakeQuantedLocation) ¶
// Should just filter indexes that are valid fmt.Println(makeQuantedLocation([]string{"Ca_%", "PMC", "Ti_%", "RTT", "filename", "Ca_int"}, []string{"1.11111", "2", "3.1415962", "5", "FileA.msa", "6"}, map[int]bool{1: true, 3: true, 4: true}))
Output: {2 5 0 FileA.msa [1.11111 3.1415962 6]} <nil>
Example (MakeROIPMCListFileContents_AB) ¶
fmt.Println(makeROIPMCListFileContents(testROIs, "5x11dataset.bin", false, false, map[int32]bool{}))
Output: 5x11dataset.bin roi1-id:7|Normal|A,15|Normal|A,388|Normal|A roi1-id:7|Normal|B,15|Normal|B,388|Normal|B roi2-id:7|Normal|A,450|Normal|A roi2-id:7|Normal|B,450|Normal|B <nil>
Example (MakeROIPMCListFileContents_AB_Dwells) ¶
fmt.Println(makeROIPMCListFileContents(testROIs, "5x11dataset.bin", false, true, map[int32]bool{15: true}))
Output: 5x11dataset.bin roi1-id:7|Normal|A,15|Normal|A,15|Dwell|A,388|Normal|A roi1-id:7|Normal|B,15|Normal|B,15|Dwell|B,388|Normal|B roi2-id:7|Normal|A,450|Normal|A roi2-id:7|Normal|B,450|Normal|B <nil>
Example (MakeROIPMCListFileContents_Combined) ¶
fmt.Println(makeROIPMCListFileContents(testROIs, "5x11dataset.bin", true, false, map[int32]bool{}))
Output: 5x11dataset.bin roi1-id:7|Normal|A,7|Normal|B,15|Normal|A,15|Normal|B,388|Normal|A,388|Normal|B roi2-id:7|Normal|A,7|Normal|B,450|Normal|A,450|Normal|B <nil>
Example (MakeROIPMCListFileContents_Combined_Dwells) ¶
fmt.Println(makeROIPMCListFileContents(testROIs, "5x11dataset.bin", true, true, map[int32]bool{15: true}))
Output: 5x11dataset.bin roi1-id:7|Normal|A,7|Normal|B,15|Normal|A,15|Normal|B,15|Dwell|A,15|Dwell|B,388|Normal|A,388|Normal|B roi2-id:7|Normal|A,7|Normal|B,450|Normal|A,450|Normal|B <nil>
Example (MatchPMCsWithDataset) ¶
l := &logger.StdOutLogger{} data := csvData{[]string{"X", "Y", "Z", "filename", "Ca_%"}, [][]string{[]string{"1", "0.40", "0", "Roastt_Laguna_Salinas_28kV_230uA_03_03_2020_111.msa", "4.5"}}} err := matchPMCsWithDataset(&data, "non-existant-file.bin", true, l) if err.Error() == "open non-existant-file.bin: no such file or directory" || err.Error() == "open non-existant-file.bin: The system cannot find the file specified." { fmt.Println("open non-existant-file.bin: Failed as expected") } fmt.Printf("%v, header[%v]=%v, data[%v]=%v\n", matchPMCsWithDataset(&data, "./testdata/LagunaSalinasdataset.bin", true, l), len(data.header)-1, data.header[5], len(data.data[0])-1, data.data[0][5]) data = csvData{[]string{"X", "Y", "Z", "filename", "Ca_%"}, [][]string{[]string{"1", "930.40", "0", "Roastt_Laguna_Salinas_28kV_230uA_03_03_2020_111.msa", "4.5"}}} fmt.Println(matchPMCsWithDataset(&data, "./testdata/LagunaSalinasdataset.bin", true, l)) data = csvData{[]string{"X", "Y", "Z", "filename", "Ca_%"}, [][]string{[]string{"1", "0.40", "0", "Roastt_Laguna_Salinas_28kV_230uA_03_03_2020_116.msa", "4.5"}}} fmt.Printf("%v, header[%v]=%v, data[%v]=%v\n", matchPMCsWithDataset(&data, "./testdata/LagunaSalinasdataset.bin", false, l), len(data.header)-1, data.header[5], len(data.data[0])-1, data.data[0][5])
Output: open non-existant-file.bin: Failed as expected <nil>, header[5]=PMC, data[5]=111 matchPMCsWithDataset Failed to match 1.00,930.40,0.00 to a PMC in dataset file <nil>, header[5]=PMC, data[5]=116
Example (ParseFloatColumnValue) ¶
fVal, err := parseFloatColumnValue("3.1415926") fmt.Printf("%v|%v\n", fVal, err) fVal, err = parseFloatColumnValue("-3.15") fmt.Printf("%v|%v\n", fVal, err) fVal, err = parseFloatColumnValue("1.234e02") fmt.Printf("%v|%v\n", fVal, err) fVal, err = parseFloatColumnValue("") fmt.Printf("%v|%v\n", fVal, err) fVal, err = parseFloatColumnValue("nan") fmt.Printf("%v|%v\n", fVal, err) fVal, err = parseFloatColumnValue("-nan") fmt.Printf("%v|%v\n", fVal, err)
Output: 3.1415925|<nil> -3.15|<nil> 123.4|<nil> 0|strconv.ParseFloat: parsing "": invalid syntax NaN|<nil> NaN|<nil>
Example (ProcessQuantROIsToPMCs_Combined_CSVRowCountROICountMismatch) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, RTT, filename 15, 5.1, 400, 7890, Normal_A_roi1-id 7, 6.1, 405, 7800, Normal_A_roi1-id 12, 6.7, 407, 7700, Normal_A_roi1-id `))), }, } fs := fileaccess.MakeS3Access(&mockS3) outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs) fmt.Printf("%v\n", err) fmt.Println(outputCSV)
Output: PMC 12 in CSV: JobData/abc123/output/node001.pmcs_result.csv doesn't exist in ROI: roi1
Example (ProcessQuantROIsToPMCs_Combined_DownloadError) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ nil, } fs := fileaccess.MakeS3Access(&mockS3) outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs) fmt.Printf("%v\n", err) fmt.Println(outputCSV)
Output: Failed to read map CSV: JobData/abc123/output/node001.pmcs_result.csv
Example (ProcessQuantROIsToPMCs_Combined_InvalidPMC) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, filename, RTT 15, 5.1, 400, Normal_A_roi1-id, 7890 Qwerty, 6.1, 405, Normal_A_roi1-id, 7800 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs) fmt.Printf("%v\n", err) fmt.Println(outputCSV)
Output: Failed to process map CSV: JobData/abc123/output/node001.pmcs_result.csv, invalid PMC Qwerty at line 4
Example (ProcessQuantROIsToPMCs_Combined_NoFileNameCol) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, RTT 15, 5.1, 400, 7890 7, 6.1, 405, 7800 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs) fmt.Printf("%v\n", err) fmt.Println(outputCSV)
Output: Map csv: JobData/abc123/output/node001.pmcs_result.csv, does not contain a filename column (used to match up ROIs)
Example (ProcessQuantROIsToPMCs_Combined_OK) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, filename, CaO_int, RTT 15, 5.1, Normal_A_roi1-id, 400, 7890 7, 6.1, Normal_B_roi2-id, 405, 7800 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs) fmt.Printf("%v\n", err) fmt.Println(outputCSV)
Output: <nil> The custom header PMC, CaO_%, filename, CaO_int, RTT 7, 5.1, Normal_A_roi1-id, 400, 7890 15, 5.1, Normal_A_roi1-id, 400, 7890 388, 5.1, Normal_A_roi1-id, 400, 7890 7, 6.1, Normal_B_roi2-id, 405, 7800 450, 6.1, Normal_B_roi2-id, 405, 7800
Example (ProcessQuantROIsToPMCs_SeparateAB_InvalidFileName) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, filename, RTT 15, 5.1, 400, Normal_A_roi1-id, 7890 15, 5.2, 401, Normal_B, 7890 7, 6.1, 405, Normal_A, 7800 7, 6.2, 406, Normal_B, 7800 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", false, testROIs) fmt.Printf("%v\n", err) fmt.Println(outputCSV)
Output: Invalid file name read: Normal_B from map CSV: JobData/abc123/output/node001.pmcs_result.csv, line 4
Example (ProcessQuantROIsToPMCs_SeparateAB_OK) ¶
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() const jobsBucket = "jobs-bucket" // Some of our files are empty, not there, have content // and they're meant to end up combined into one response... mockS3.ExpGetObjectInput = []s3.GetObjectInput{ { Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"), }, } mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{ { Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1 PMC, CaO_%, CaO_int, filename, RTT 15, 5.1, 400, Normal_A_roi1-id, 7890 15, 5.2, 401, Normal_B_roi1-id, 7890 7, 6.1, 405, Normal_A_roi2-id, 7800 7, 6.2, 406, Normal_B_roi2-id, 7800 `))), }, } fs := fileaccess.MakeS3Access(&mockS3) outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", false, testROIs) fmt.Printf("%v\n", err) fmt.Println(outputCSV)
Output: <nil> The custom header PMC, CaO_%, CaO_int, filename, RTT 7, 5.1, 400, Normal_A_roi1-id, 7890 7, 5.2, 401, Normal_B_roi1-id, 7890 15, 5.1, 400, Normal_A_roi1-id, 7890 15, 5.2, 401, Normal_B_roi1-id, 7890 388, 5.1, 400, Normal_A_roi1-id, 7890 388, 5.2, 401, Normal_B_roi1-id, 7890 7, 6.1, 405, Normal_A_roi2-id, 7800 7, 6.2, 406, Normal_B_roi2-id, 7800 450, 6.1, 405, Normal_A_roi2-id, 7800 450, 6.2, 406, Normal_B_roi2-id, 7800
Example (ReadCSV) ¶
csv := `something header more header col 1,"col, 2", col_3 "value one",123, 456 value two,444,555 ` d, err := readCSV(csv, 2) fmt.Printf("%v|%v", d, err)
Output: {[col 1 col, 2 col_3] [[value one 123 456] [value two 444 555]]}|<nil>
Example (StageQuant) ¶
Stage the quantification ready for uploading to OCS
var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() datasetsbucket := "datasets-bucket" mockS3.ExpCopyObjectInput = []s3.CopyObjectInput{ { Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/Staging/mydatasetid/myjobid.csv"), CopySource: aws.String(datasetsbucket + "/UserContent/shared/mydatasetid/Quantifications/myjobid.csv"), }, { Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/Staging/mydatasetid/summary-myjobid.json"), CopySource: aws.String(datasetsbucket + "/UserContent/shared/mydatasetid/Quantifications/summary-myjobid.json"), }, } mockS3.QueuedCopyObjectOutput = []*s3.CopyObjectOutput{ {}, {}, } datasetid := "mydatasetid" jobid := "myjobid" products := ProductSet{ OcsPath: "", SourceBucket: "", SourcePrefix: "", DatasetID: "", JobID: "", PqrFileName: "myjobid.csv", PqrMetaFileName: "myjobid.csv.met", PqpFileName: "summary-myjobid.json", PqpMetaFileName: "summary-myjobid.json.met", } fs := fileaccess.MakeS3Access(&mockS3) err := stageQuant(fs, datasetsbucket, datasetid, jobid, products) if err != nil { fmt.Printf("%v", err) }
Output:
Example (ValidateParameters) ¶
fmt.Printf("%v\n", validateParameters("-b,0,50,2,10 -f")) fmt.Printf("%v\n", validateParameters("-b,0,50,2,10.55 -o \"filename.whatever\" -f -Fe,1")) fmt.Printf("%v\n", validateParameters("-b,0,50,2,10;ls -al;echo -f")) fmt.Printf("%v\n", validateParameters("-b,0,50,2,10&&rm -rf ~/; -f"))
Output: <nil> <nil> Invalid parameters passed: -b,0,50,2,10;ls -al;echo -f Invalid parameters passed: -b,0,50,2,10&&rm -rf ~/; -f
Index ¶
- Constants
- func CheckQuantificationNameExists(name string, datasetID string, userID string, svcs *services.APIServices) bool
- func ConvertQuantificationCSV(logger logger.ILogger, data string, expectMetaColumns []string, ...) ([]byte, []string, error)
- func CreateJob(svcs *services.APIServices, createParams JobCreateParams, wg *sync.WaitGroup) (string, error)
- func GetBlessedQuantFile(svcs *services.APIServices, datasetID string) (BlessFile, *BlessFileItem, string, error)
- func GetQuantColumnIndex(quant *protos.Quantification, column string) int32
- func GetQuantification(svcs *services.APIServices, s3Path string) (*protos.Quantification, error)
- func GetWeightPercentColumnsInQuant(quant *protos.Quantification) []string
- func ImportQuantCSV(svcs *services.APIServices, datasetID string, importUser pixlUser.UserInfo, ...) (string, error)
- func PublishQuant(fs fileaccess.FileAccess, config PublisherConfig, creator pixlUser.UserInfo, ...) error
- func ReadQuantificationFile(path string) (*protos.Quantification, error)
- func ShareQuantification(svcs *services.APIServices, userID string, datasetID string, jobID string) error
- type BlessFile
- type BlessFileItem
- type DetectorCode
- type JobCreateParams
- type JobStartingParameters
- type JobStartingParametersWithPMCCount
- type JobStartingParametersWithPMCs
- type JobStatus
- type JobStatusValue
- type JobSummaryItem
- func GetJobSummary(fs fileaccess.FileAccess, bucket string, userID string, datasetID string, ...) (JobSummaryItem, error)
- func ListQuantJobsForDataset(svcs *services.APIServices, userID string, datasetID string) ([]JobSummaryItem, error)
- func SetMissingSummaryFields(summary JobSummaryItem) JobSummaryItem
- type JobSummaryMap
- type OcsMetaData
- type PiquantParams
- type ProductSet
- type ProductType
- type Publication
- type PublicationRecord
- type Publications
- type PublisherConfig
- type ROIWithPMCs
Examples ¶
- Package (CheckExistingQuants)
- Package (CheckExistingQuantsUnsorted)
- Package (CleanLogName)
- Package (CombineQuantOutputs_BadPMC)
- Package (CombineQuantOutputs_DownloadError)
- Package (CombineQuantOutputs_DuplicatePMC)
- Package (CombineQuantOutputs_LastLineCutOff)
- Package (CombineQuantOutputs_OK)
- Package (ConvertQuantificationData)
- Package (DecodeMapFileNameColumn)
- Package (EstimateNodeCount)
- Package (FilesPerNode)
- Package (FilterListItems)
- Package (GenerateMetFiles)
- Package (GetElements)
- Package (GetInterestingColIndexes)
- Package (MakeColumnTypeList)
- Package (MakeIndividualPMCListFileContents_AB)
- Package (MakeIndividualPMCListFileContents_AB_Dwell)
- Package (MakeIndividualPMCListFileContents_Combined)
- Package (MakeIndividualPMCListFileContents_Combined_Dwell)
- Package (MakeJobStartingParametersWithPMCCount)
- Package (MakeQuantJobPMCLists)
- Package (MakeQuantProducts)
- Package (MakeQuantedLocation)
- Package (MakeROIPMCListFileContents_AB)
- Package (MakeROIPMCListFileContents_AB_Dwells)
- Package (MakeROIPMCListFileContents_Combined)
- Package (MakeROIPMCListFileContents_Combined_Dwells)
- Package (MatchPMCsWithDataset)
- Package (ParseFloatColumnValue)
- Package (ProcessQuantROIsToPMCs_Combined_CSVRowCountROICountMismatch)
- Package (ProcessQuantROIsToPMCs_Combined_DownloadError)
- Package (ProcessQuantROIsToPMCs_Combined_InvalidPMC)
- Package (ProcessQuantROIsToPMCs_Combined_NoFileNameCol)
- Package (ProcessQuantROIsToPMCs_Combined_OK)
- Package (ProcessQuantROIsToPMCs_SeparateAB_InvalidFileName)
- Package (ProcessQuantROIsToPMCs_SeparateAB_OK)
- Package (ReadCSV)
- Package (StageQuant)
- Package (ValidateParameters)
Constants ¶
const ( JobStarting JobStatusValue = "starting" JobPreparingNodes = "preparing_nodes" JobNodesRunning = "nodes_running" JobGatheringResults = "gathering_results" JobComplete = "complete" JobError = "error" )
Valid job status strings
const JobParamsFileName = "params.json"
JobParamsFileName - File name of job params file
const OcsPath string = "/ods/surface/sol/XXXXX/soas/rdr/pixl/PQA"
OcsPath directs publishing to ODS
const OcsStagingPath string = "Publish/Staging"
OcsStagingPath path in s3 bucket where files are staged before publishing to ODS
const PublicationsPath string = "Publish/publications.json"
PublicationsPath path in s3 bucket where the list of publication records are kept
const QuantModeABManualUpload = "ABManual"
const QuantModeABMultiQuant = "ABMultiQuant"
const QuantModeCombinedAB = "Combined"
const QuantModeCombinedABBulk = "CombinedBulk"
const QuantModeCombinedManualUpload = "ABManual"
const QuantModeCombinedMultiQuant = "CombinedMultiQuant"
const QuantModeSeparateAB = "AB"
const QuantModeSeparateABBulk = "ABBulk"
Variables ¶
This section is empty.
Functions ¶
func ConvertQuantificationCSV ¶
func ConvertQuantificationCSV(logger logger.ILogger, data string, expectMetaColumns []string, matchPMCDatasetFileName string, matchPMCByCoord bool, detectorIDOverride string, detectorDuplicateAB bool) ([]byte, []string, error)
ConvertQuantificationCSV - converts from incoming string CSV data to serialised binary data Returns the serialised quantification bytes and the elements that were quantified
func CreateJob ¶
func CreateJob(svcs *services.APIServices, createParams JobCreateParams, wg *sync.WaitGroup) (string, error)
CreateJob - creates a new quantification job
func GetBlessedQuantFile ¶
func GetBlessedQuantFile(svcs *services.APIServices, datasetID string) (BlessFile, *BlessFileItem, string, error)
Downloads & parses the blessed quants file. Returns: - the parsed contents - the blessed quant job info (BlessItem) - the path (in case we want to update the same file) - error or nil
func GetQuantColumnIndex ¶
func GetQuantColumnIndex(quant *protos.Quantification, column string) int32
GetQuantColumnIndex - returns index of column in quantification or -1 if not found
func GetQuantification ¶
func GetQuantification(svcs *services.APIServices, s3Path string) (*protos.Quantification, error)
func GetWeightPercentColumnsInQuant ¶
func GetWeightPercentColumnsInQuant(quant *protos.Quantification) []string
GetWeightPercentColumnsInQuant - returns weight % columns, ones ending in _%
func ImportQuantCSV ¶
func PublishQuant ¶
func PublishQuant(fs fileaccess.FileAccess, config PublisherConfig, creator pixlUser.UserInfo, log logger.ILogger, datasetID, jobID string, notifications notifications.NotificationManager) error
func ReadQuantificationFile ¶
func ReadQuantificationFile(path string) (*protos.Quantification, error)
func ShareQuantification ¶
Types ¶
type BlessFile ¶
type BlessFile struct {
History []BlessFileItem `json:"history"`
}
type BlessFileItem ¶
type DetectorCode ¶
type DetectorCode string
DetectorCode Indicates the detector used in a Quantification; A/B or combined
const ( Combined DetectorCode = "C" // Combined uses the combined detector for quantification Separate = "S" // Separate uses the separate A/B detectors for quantification )
type JobCreateParams ¶
type JobCreateParams struct { Name string `json:"name"` DatasetPath string `json:"datasetPath"` PMCs []int32 `json:"pmcs"` Elements []string `json:"elements"` DetectorConfig string `json:"detectorconfig"` Parameters string `json:"parameters"` RunTimeSec int32 `json:"runtimesec"` RoiID string `json:"roiID"` // There is now a list of ROI IDs that can be provided too. More relevant with the QuantMode *Bulk options ElementSetID string `json:"elementSetID"` DatasetID string `json:"dataset_id"` Creator pixlUser.UserInfo `json:"creator"` QuantMode string `json:"quantMode"` RoiIDs []string `json:"roiIDs"` // If QuantMode = *Bulk, this is used, pmcs is ignored. IncludeDwells bool `json:"includeDwells,omitempty"` Command string `json:"command,omitempty"` }
JobCreateParams - Parameters to CreateJob()
type JobStartingParameters ¶
type JobStartingParameters struct { Name string `json:"name"` DataBucket string `json:"dataBucket"` DatasetPath string `json:"datasetPath"` DatasetID string `json:"datasetID"` PiquantJobsBucket string `json:"jobBucket"` DetectorConfig string `json:"detectorConfig"` Elements []string `json:"elements"` Parameters string `json:"parameters"` RunTimeSec int32 `json:"runTimeSec"` CoresPerNode int32 `json:"coresPerNode"` StartUnixTime int64 `json:"startUnixTime"` Creator pixlUser.UserInfo `json:"creator"` RoiID string `json:"roiID"` ElementSetID string `json:"elementSetID"` PIQUANTVersion string `json:"piquantVersion"` QuantMode string `json:"quantMode"` Comments string `json:"comments"` RoiIDs []string `json:"roiIDs"` IncludeDwells bool `json:"includeDwells,omitempty"` Command string `json:"command,omitempty"` }
JobStartingParameters - parameters to start the job, saved in S3 as JobParamsFileName for the job
type JobStartingParametersWithPMCCount ¶
type JobStartingParametersWithPMCCount struct { PMCCount int32 `json:"pmcsCount"` *JobStartingParameters }
JobStartingParametersWithPMCCount - summary version of JobStartingParametersWithPMCs, only storing PMC count
func MakeJobStartingParametersWithPMCCount ¶
func MakeJobStartingParametersWithPMCCount(params JobStartingParametersWithPMCs) JobStartingParametersWithPMCCount
MakeJobStartingParametersWithPMCCount - Converting full to summary version of struct
type JobStartingParametersWithPMCs ¶
type JobStartingParametersWithPMCs struct { PMCs []int32 `json:"pmcs"` *JobStartingParameters }
JobStartingParametersWithPMCs - When saving file with name: JobParamsFileName, we save all PMCs, but in further references to the job parameters, eg in job list summary (filepaths.JobSummarySuffix), we only store the PMC count
type JobStatus ¶
type JobStatus struct { JobID string `json:"jobId"` Status JobStatusValue `json:"status"` Message string `json:"message"` EndUnixTime int64 `json:"endUnixTime"` OutputFilePath string `json:"outputFilePath"` PiquantLogList []string `json:"piquantLogList"` }
JobStatus - job status that gets saved to S3 as <job-id>JobStatusSuffix as it progresses
type JobSummaryItem ¶
type JobSummaryItem struct { Params JobStartingParametersWithPMCCount `json:"params"` Elements []string `json:"elements"` *JobStatus }
JobSummaryItem all metadata stored for an individual job/quant file (even after it was generated)
func GetJobSummary ¶
func GetJobSummary(fs fileaccess.FileAccess, bucket string, userID string, datasetID string, jobID string) (JobSummaryItem, error)
GetJobSummary - Get the summary information from a quant job json file in S3
func ListQuantJobsForDataset ¶
func ListQuantJobsForDataset(svcs *services.APIServices, userID string, datasetID string) ([]JobSummaryItem, error)
Downloads job summary file with path: s3://PiquantJobsBucket/RootJobSummaries/<dataset-id>JobSummarySuffix Returns only jobs created by userID that are NOT complete (still in progress/failed)
func SetMissingSummaryFields ¶
func SetMissingSummaryFields(summary JobSummaryItem) JobSummaryItem
SetMissingSummaryFields - ensure the fields all exist over time.
type JobSummaryMap ¶
type JobSummaryMap map[string]JobSummaryItem
JobSummaryMap - saved by job updater lambda function, read by quant listing handler
type OcsMetaData ¶
type OcsMetaData struct {
Description string `json:"description"`
}
OcsMetaData metadata file that provides information to OCS about published Datasets
type PiquantParams ¶
type PiquantParams struct { RunTimeEnv string `json:"runtimeEnv"` JobID string `json:"jobId"` JobsPath string `json:"jobsPath"` DatasetPath string `json:"datasetPath"` DetectorConfig string `json:"detectorConfig"` Elements []string `json:"elements"` Parameters string `json:"parameters"` DatasetsBucket string `json:"datasetsBucket"` ConfigBucket string `json:"configBucket"` PiquantJobsBucket string `json:"jobBucket"` QuantName string `json:"name"` PMCListName string `json:"pmcListName"` Command string `json:"command"` }
PiquantParams - Parameters for running piquant, as generated by PIXLISE API
type ProductSet ¶
type ProductType ¶
type ProductType string
ProductType Indicates the Product Type to publish to OCS
const ( PiQuantResults ProductType = "PQR" // PiQuantResults PQR Products contain the PiQuant Quantification Results PiQuantParams ProductType = "PQP" // PiQuantParams PQP Products contain the PiQuant Runtime Parameters )
type Publication ¶
type Publication struct { Version int `json:"version"` PublicationTime time.Time `json:"timestamp"` Publisher string `json:"publisher"` }
Publication keeps details about a particular publication
type PublicationRecord ¶
type PublicationRecord struct { DatasetID string `json:"dataset-id"` JobID string `json:"job-id"` Publications []Publication `json:"publications"` }
PublicationRecord keeps tracks all the publications for a quantification
type Publications ¶
type Publications struct {
Datasets []PublicationRecord `json:"datasets"`
}
Publications keeps a list of publications for all Datasets
type PublisherConfig ¶
type ROIWithPMCs ¶
type ROIWithPMCs struct { PMCs []int ID string *roiModel.ROISavedItem }