quantModel

package
v3.12.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 22, 2023 License: Apache-2.0 Imports: 37 Imported by: 0

Documentation

Overview

Exposes interfaces and structures required to run PIQUANT in the Kubernetes cluster along with functions to access quantification files, logs, results and summaries of quant jobs.

Example (CheckExistingQuants)

This Example demonstrates how to check for existing published quants. 

var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

datasetsbucket := "datasets-bucket"
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/publications.json"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`
{
	"Datasets": [
		{
			"dataset-id": "mydataset",
			"job-id": "blah",
			"publications": [
				{
					"publisher": "abc",
					"version": 1,
					"timestamp": "2018-09-21T12:42:31Z"
				},
				{
					"publisher": "abc",
					"version": 2,
					"timestamp": "2018-09-22T12:42:31Z"
				}
			]
		}
	]
}
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)

// Check for the latest version of the published quant. It should be version 2.
i, err := checkCurrentlyPublishedQuantVersion(fs, datasetsbucket, "mydataset")
if err != nil {
	fmt.Printf("%v\n", err)
}

fmt.Printf("Version found: %v\n", i)

Output:

Version found: 2
Example (CheckExistingQuantsUnsorted)

This Example we demonstrate sort capabilities for unordered publication times. 

var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

datasetsbucket := "datasets-bucket"
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/publications.json"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`
{
	"Datasets": [
		{
			"dataset-id": "mydataset",
			"job-id": "blah",
			"publications": [
				{
					"publisher": "abc",
					"version": 2,
					"timestamp": "2018-09-22T12:42:31Z"
				},
				{
					"publisher": "abc",
					"version": 1,
					"timestamp": "2018-09-21T12:42:31Z"
				}
			]
		}
	]
}
`))),
	},
}
fs := fileaccess.MakeS3Access(&mockS3)

// Check for the latest version of the published quant. It should be version 2. But we have deliberately switched
// the order of the publications to ensure it doesn't match the last in the list.
i, err := checkCurrentlyPublishedQuantVersion(fs, datasetsbucket, "mydataset")
if err != nil {
	fmt.Printf("%v\n", err)
}

fmt.Printf("Version found: %v\n", i)

Output:

Version found: 2
Example (CleanLogName) 
// Don't fix it...
fmt.Println(cleanLogName("node00001_data.log"))
// Do fix it...
fmt.Println(cleanLogName("node00001.pmcs_stdout.log"))
// Do fix it...
fmt.Println(cleanLogName("NODE00001.PMCS_stdout.log"))

Output:

node00001_data.log
node00001_stdout.log
NODE00001_stdout.log
Example (CombineQuantOutputs_BadPMC) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, RTT
30, 5.1, 400, 7890
12, 6.1, 405, 7800
`))),
	},
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row2
PMC, CaO_%, CaO_int, RTT
NaN, 7.1, 415, 7840
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"})

fmt.Printf("%v\n", err)
fmt.Println(combinedCSV)

Output:

Failed to combine map segment: JobData/abc123/output/node002.pmcs_result.csv, invalid PMC NaN at line 3
Example (CombineQuantOutputs_DownloadError) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, RTT
30, 5.1, 400, 7890
12, 6.1, 405, 7800
`))),
	},
	nil,
}

fs := fileaccess.MakeS3Access(&mockS3)
combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"})

fmt.Printf("%v\n", err)
fmt.Println(combinedCSV)

Output:

Failed to combine map segment: JobData/abc123/output/node002.pmcs_result.csv
Example (CombineQuantOutputs_DuplicatePMC) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"),
	},
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node003.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, RTT
30, 5.1, 400, 7890
12, 6.1, 405, 7800
`))),
	},
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row2
PMC, CaO_%, CaO_int, RTT
18, 7.1, 415, 7840
`))),
	},
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row3
PMC, CaO_%, CaO_int, RTT
3, 1.1, 450, 7830
30, 1.3, 451, 7833
40, 8.1, 455, 7870
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"})

fmt.Printf("%v\n", err)
fmt.Println(combinedCSV)

Output:

<nil>
The custom header
PMC, CaO_%, CaO_int, RTT
3, 1.1, 450, 7830
12, 6.1, 405, 7800
18, 7.1, 415, 7840
30, 5.1, 400, 7890
30, 1.3, 451, 7833
40, 8.1, 455, 7870
Example (CombineQuantOutputs_LastLineCutOff) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, RTT
30, 5.1, 400, 7890
12, 6.1, 405, 7800
`))),
	},
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row2
PMC, CaO_%, CaO_int, RTT
31
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"})

fmt.Printf("%v\n", err)
fmt.Println(combinedCSV)

Output:

Failed to combine map segment: JobData/abc123/output/node002.pmcs_result.csv, no PMC at line 3
Example (CombineQuantOutputs_OK) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node002.pmcs_result.csv"),
	},
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node003.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, RTT
30, 5.1, 400, 7890
12, 6.1, 405, 7800
`))),
	},
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row2
PMC, CaO_%, CaO_int, RTT
18, 7.1, 415, 7840
`))),
	},
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row3
PMC, CaO_%, CaO_int, RTT
3, 1.1, 450, 7830
40, 8.1, 455, 7870
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
combinedCSV, err := combineQuantOutputs(fs, jobsBucket, "JobData/abc123", "The custom header", []string{"node001.pmcs", "node002.pmcs", "node003.pmcs"})

fmt.Printf("%v\n", err)
fmt.Println(combinedCSV)

Output:

<nil>
The custom header
PMC, CaO_%, CaO_int, RTT
3, 1.1, 450, 7830
12, 6.1, 405, 7800
18, 7.1, 415, 7840
30, 5.1, 400, 7890
40, 8.1, 455, 7870
Example (ConvertQuantificationData) 
data := csvData{
	[]string{"PMC", "Ca_%", "Ca_int", "SCLK", "Ti_%", "filename", "RTT"},
	[][]string{
		[]string{"23", "1.5", "5", "11111", "4", "fileA.msa", "44"},
		[]string{"70", "3.4", "32", "12345", "4.21", "fileB.msa", "45"},
	},
}

result, err := convertQuantificationData(data, []string{"PMC", "RTT", "SCLK", "filename"})
fmt.Printf("%v|%v\n", result, err)

Output:

{[Ca_% Ca_int Ti_%] [F I F] [{23 44 11111 fileA.msa [1.5 5 4]} {70 45 12345 fileB.msa [3.4 32 4.21]}]}|<nil>
Example (DecodeMapFileNameColumn) 
rt, det, err := decodeMapFileNameColumn("file.txt")
fmt.Printf("%v|%v|%v\n", rt, det, err)

rt, det, err = decodeMapFileNameColumn("Normal_A")
fmt.Printf("%v|%v|%v\n", rt, det, err)

rt, det, err = decodeMapFileNameColumn("Normal_A_MyRoiID")
fmt.Printf("%v|%v|%v\n", rt, det, err)

rt, det, err = decodeMapFileNameColumn("Dwell_B")
fmt.Printf("%v|%v|%v\n", rt, det, err)

rt, det, err = decodeMapFileNameColumn("Normal_C")
fmt.Printf("%v|%v|%v\n", rt, det, err)

rt, det, err = decodeMapFileNameColumn("LongRead_B")
fmt.Printf("%v|%v|%v\n", rt, det, err)

rt, det, err = decodeMapFileNameColumn("Scotland_something_00012.msa")
fmt.Printf("%v|%v|%v\n", rt, det, err)

rt, det, err = decodeMapFileNameColumn("Scotland_something_00012_10keV_33.msa")
fmt.Printf("%v|%v|%v\n", rt, det, err)

rt, det, err = decodeMapFileNameColumn("Normal_A_0123456789_873495_455.msa")
fmt.Printf("%v|%v|%v\n", rt, det, err)

Output:

||decodeMapFileNameColumn: Invalid READTYPE in filename: "file.txt"
Normal|A|<nil>
Normal|A|<nil>
Dwell|B|<nil>
||decodeMapFileNameColumn: Invalid DETECTOR_ID in filename: "Normal_C"
||decodeMapFileNameColumn: Invalid READTYPE in filename: "LongRead_B"
||decodeMapFileNameColumn: Invalid READTYPE in filename: "Scotland_something_00012.msa"
||decodeMapFileNameColumn: Invalid READTYPE in filename: "Scotland_something_00012_10keV_33.msa"
Normal|A|<nil>
Example (EstimateNodeCount)

5x11, 4035 PMCs, 9 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:23 (with params) 

=> 4035*2=8070 spectra on 160 cores in 203 sec => 50.44 spectra/core in 203 sec => 4.02sec/spectra

5x11, 4035 PMCs, 6 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:47 (with params) 

=> 4035*2=8070 spectra on 160 cores in 167 sec => 50.44 spectra/core in 167 sec => 3.31sec/spectra

5x11, 4035 PMCs, 3 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 1:52 (with params) 

=> 4035*2=8070 spectra on 160 cores in 112 sec => 50.44 spectra/core in 112 sec => 2.22sec/spectra

5x11, 4035 PMCs, 3 elements, 8 cores (7.5 allocation in kubernetes), 10 nodes. Runtime: 3:32 (with params) 

=> 4035*2=8070 spectra on 80 cores in 212 sec => 100.88 spectra/core in 212 sec => 2.10 sec/spectra

3 elements => 2.10sec/spectra 3 elements => 2.22sec/spectra 

3 elems jumped 1.09sec

6 elements => 3.31sec/spectra 

3 elems jumped 0.71sec

9 elements => 4.02sec/spectra

Assumptions:

  • Lets make this calcatable: 9elem=4sec/spectra, 3elem = 2sec/spectra, linearly interpolate in this range

  • Works out to elements = 3*sec - 3

  • To calculate node count, we are given Core count, Runtime desired, Spectra count, Element count

  • Using the above: Runtime = Spectra*SpectraRuntime / (Core*Nodes) Nodes = Spectra*SpectraRuntime / (Runtime * Core)

    SpectraRuntime is calculated using the above formula: Elements = 3 * Sec - 3 SpectraRuntime = (Elements+3) / 3

    Nodes = Spectra*((Elements + 3) / 3) / (RuntimeDesired * Cores) Nodes = Spectra*(Elements+3) / 3*(RuntimeDesired * Cores)

    Example using the values from above: Nodes = 8070*(3+3)/(3*120*8) Nodes = 8070*6/5088 = 9.5, close to 10

    Nodes = 8070*(9+3)/(3*203*8) Nodes = 96840 / 4872 = 19.9, close to 20

    Nodes = 8070*(6+3)/(3*167*8) Nodes = 72630 / 4008 = 18.12, close to 20

    If we're happy to run 6 elems, 8070 spectra, 8 cores in 5 minutes: Nodes = 8070*(6+3) / (3*300*8) Nodes = 72630 / 7200 = 10 nodes... seems reasonable 

// Based on experimental runs in: https://github.com/pixlise/core/-/issues/113

// Can only use the ones where we had allcoation of 7.5 set in kubernetes, because the others weren't maxing out cores

// 5x11, 4035 PMCs, 3 elements, 8 cores (7.5 allocation in kubernetes), 10 nodes. Runtime: 3:22
fmt.Println(estimateNodeCount(4035*2, 3, 3*60+22, 8, 50))
// 5x11, 4035 PMCs, 3 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 1:52 (with params)
fmt.Println(estimateNodeCount(4035*2, 3, 60+52, 8, 50))
// 5x11, 4035 PMCs, 4 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:11 (with params)
fmt.Println(estimateNodeCount(4035*2, 4, 2*60+11, 8, 50))
// 5x11, 4035 PMCs, 5 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:26 (with params)
fmt.Println(estimateNodeCount(4035*2, 5, 2*60+26, 8, 50))
// 5x11, 4035 PMCs, 6 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:47 (with params)
fmt.Println(estimateNodeCount(4035*2, 6, 2*60+47, 8, 50))
// 5x11, 4035 PMCs, 7 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 2:55 (no params though)
fmt.Println(estimateNodeCount(4035*2, 7, 2*60+55, 8, 50))
// 5x11, 4035 PMCs, 8 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:12 (with params)
fmt.Println(estimateNodeCount(4035*2, 8, 3*60+12, 8, 50))
// 5x11, 4035 PMCs, 9 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:23 (with params)
fmt.Println(estimateNodeCount(4035*2, 9, 3*60+23, 8, 50))
// 5x11, 4035 PMCs, 10 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:35 (with params)
fmt.Println(estimateNodeCount(4035*2, 10, 3*60+35, 8, 50))
// 5x11, 4035 PMCs, 11 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 3:46 (with params)
fmt.Println(estimateNodeCount(4035*2, 11, 3*60+46, 8, 50))

// 5x5, 1769 PMCs, 11 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 1:47 (with params)
fmt.Println(estimateNodeCount(1769*2, 11, 60+47, 8, 50))

// 5x5, 1769 PMCs, 4 elements, 8 cores (7.5 allocation in kubernetes), 20 nodes. Runtime: 0:59 (with params)
fmt.Println(estimateNodeCount(1769*2, 4, 59, 8, 50))

// Ensure the max cores have an effect
fmt.Println(estimateNodeCount(1769*2, 4, 59, 8, 6))

// It's a bit unfortunate we ran all but 1 tests on the same number of cores, but
// the above data varies the spectra count, element count and expected runtime

// Below we'd expect 20 for all answers except the first one, but there's a slight (and
// allowable) drift because we're not exactly spot on with our estimate, and there's
// fixed overhead time we aren't even calculating properly

Output:

10
18
18
18
18
19
19
20
20
21
19
17
6
Example (FilesPerNode) 
fmt.Println(filesPerNode(8088, 5))
fmt.Println(filesPerNode(8068, 3))

Output:

1619
2690
Example (FilterListItems) 
// Should just filter indexes that are valid
idxToIgnoreMap := map[int]bool{
	-9: true,
	1:  true,
	2:  true,
	5:  true,
	6:  true,
}

fmt.Println(filterListItems([]string{"snowboarding", "is", "awesome", "says", "Peter", "Nemere"}, idxToIgnoreMap))

Output:

[snowboarding says Peter]
Example (GenerateMetFiles) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

datasetsbucket := "datasets-bucket"

mockS3.ExpPutObjectInput = []s3.PutObjectInput{
	{
		Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/Staging/mydatasetid/myjobid.csv.met"), Body: bytes.NewReader([]byte(`{
    "description": "PiQuant Results File"
}`)),
	},
	{
		Bucket: aws.String(datasetsbucket), Key: aws.String("Publish/Staging/mydatasetid/summary-myjobid.json.met"), Body: bytes.NewReader([]byte(`{
    "description": "PiQuant Runtime Parameters File"
}`)),
	},
}
mockS3.QueuedPutObjectOutput = []*s3.PutObjectOutput{
	{},
	{},
}

datasetid := "mydatasetid"
jobid := "myjobid"
products := ProductSet{
	OcsPath:         "",
	SourceBucket:    "",
	SourcePrefix:    "",
	DatasetID:       datasetid,
	JobID:           jobid,
	PqrFileName:     "myjobid.csv",
	PqrMetaFileName: "myjobid.csv.met",
	PqpFileName:     "summary-myjobid.json",
	PqpMetaFileName: "summary-myjobid.json.met",
}
fs := fileaccess.MakeS3Access(&mockS3)
err := stageMetFiles(fs, datasetsbucket, products)
if err != nil {
	fmt.Printf("%v", err)
}

Output:
Example (GetElements) 
fmt.Printf("%v", getElements([]string{"PMC", "SCLK", "Ca_%", "Ti_%", "Ca_int", "Ti_int", "livetime", "Mg_%", "chisq"}))

Output:

[Ca Ti Mg]
Example (GetInterestingColIndexes) 
header := []string{"PMC", "K_%", "Ca_%", "Fe_%", "K_int", "Ca_int", "Fe_int", "K_err", "Ca_err", "Fe_err", "total_counts", "livetime", "chisq", "eVstart", "eV/ch", "res", "iter", "filename", "Events", "Triggers", "SCLK", "RTT"}
interesting, err := getInterestingColIndexes(header, []string{"PMC", "filename", "SCLK", "RTT"})
fmt.Printf("\"%v\" \"%v\"\n", interesting, err)
interesting, err = getInterestingColIndexes(header, []string{"K_%", "total_counts"})
fmt.Printf("\"%v\" \"%v\"\n", interesting, err)

// Bad cases
interesting, err = getInterestingColIndexes(header, []string{"PMC", "TheFileName", "SCLK", "RTT"})
fmt.Printf("\"%v\" \"%v\"\n", interesting, err)
header[5] = "SCLK"
interesting, err = getInterestingColIndexes(header, []string{"PMC", "TheFileName", "SCLK", "RTT"})
fmt.Printf("\"%v\" \"%v\"\n", interesting, err)

// 22 header items...

Output:

"map[PMC:0 RTT:21 SCLK:20 filename:17]" "<nil>"
"map[K_%:1 total_counts:10]" "<nil>"
"map[]" "CSV column missing: TheFileName"
"map[]" "Duplicate CSV column: SCLK"
Example (MakeColumnTypeList) 
data := csvData{[]string{"a", "b", "c", "d", "e"}, [][]string{[]string{"1.11111", "2", "3.1415962", "5", "6"}}}
result, err := makeColumnTypeList(data, map[int]bool{2: true, 3: true})
fmt.Printf("%v|%v\n", result, err)
result, err = makeColumnTypeList(data, map[int]bool{})
fmt.Printf("%v|%v\n", result, err)

// Bad type
data = csvData{[]string{"a", "b", "c", "d", "e"}, [][]string{[]string{"1.11111", "Wanaka", "3.1415962", "5"}}}
result, err = makeColumnTypeList(data, map[int]bool{2: true, 3: true})
fmt.Printf("%v|%v\n", result, err)

// Skipping the string 1 should make it work...
result, err = makeColumnTypeList(data, map[int]bool{1: true, 3: true})
fmt.Printf("%v|%v\n", result, err)

Output:

[F I I]|<nil>
[F I F I I]|<nil>
[F]|Failed to parse "Wanaka" as float or int at col 1/row 0
[F F]|<nil>
Example (MakeIndividualPMCListFileContents_AB) 
fmt.Println(makeIndividualPMCListFileContents([]int32{15, 7, 388}, "5x11dataset.bin", false, false, map[int32]bool{}))

Output:

5x11dataset.bin
15|Normal|A
15|Normal|B
7|Normal|A
7|Normal|B
388|Normal|A
388|Normal|B
 <nil>
Example (MakeIndividualPMCListFileContents_AB_Dwell) 
fmt.Println(makeIndividualPMCListFileContents([]int32{15, 7, 388}, "5x11dataset.bin", false, true, map[int32]bool{15: true}))

Output:

5x11dataset.bin
15|Normal|A,15|Dwell|A
15|Normal|B,15|Dwell|B
7|Normal|A
7|Normal|B
388|Normal|A
388|Normal|B
 <nil>
Example (MakeIndividualPMCListFileContents_Combined) 
fmt.Println(makeIndividualPMCListFileContents([]int32{15, 7, 388}, "5x11dataset.bin", true, false, map[int32]bool{}))

Output:

5x11dataset.bin
15|Normal|A,15|Normal|B
7|Normal|A,7|Normal|B
388|Normal|A,388|Normal|B
 <nil>
Example (MakeIndividualPMCListFileContents_Combined_Dwell) 
fmt.Println(makeIndividualPMCListFileContents([]int32{15, 7, 388}, "5x11dataset.bin", true, true, map[int32]bool{15: true}))

Output:

5x11dataset.bin
15|Normal|A,15|Normal|B,15|Dwell|A,15|Dwell|B
7|Normal|A,7|Normal|B
388|Normal|A,388|Normal|B
 <nil>
Example (MakeJobStartingParametersWithPMCCount)

Making sure our embedded structure copying works 

in := JobStartingParametersWithPMCs{
	[]int32{4, 59, 444, 2313, 329},
	&JobStartingParameters{
		Name:       "name",
		DataBucket: "databucket",
		//DatasetsBucket:    "datasetsbucket",
		//ConfigBucket:      "configbucket",
		DatasetPath:       "datasetpath",
		DatasetID:         "datasetid",
		PiquantJobsBucket: "jobbucket",
		DetectorConfig:    "config",
		Elements:          []string{"Ti", "Al", "Ca"},
		Parameters:        "params",
		RunTimeSec:        39,
		CoresPerNode:      3,
		StartUnixTime:     33332222,
		Creator: pixlUser.UserInfo{
			Name:        "creator",
			UserID:      "creator-id-123",
			Email:       "niko@rockstar.com",
			Permissions: map[string]bool{"read:something": true, "read:another": true},
		},
		RoiID:          "roiID",
		ElementSetID:   "elemSetID",
		PIQUANTVersion: "3.0.3",
		Command:        "map",
	},
}

out := MakeJobStartingParametersWithPMCCount(in)
fmt.Printf("pmc=%v, name=%v, elements=%v", out.PMCCount, out.Name, out.Elements)

Output:

pmc=5, name=name, elements=[Ti Al Ca]
Example (MakeQuantJobPMCLists) 
fmt.Println(makeQuantJobPMCLists([]int32{1, 2, 3, 4, 5, 6, 7, 8}, 3))
fmt.Println(makeQuantJobPMCLists([]int32{1, 2, 3, 4, 5, 6, 7, 8, 9}, 3))
fmt.Println(makeQuantJobPMCLists([]int32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 3))

Output:

[[1 2 3] [4 5 6] [7 8]]
[[1 2 3] [4 5 6] [7 8 9]]
[[1 2 3] [4 5 6] [7 8 9] [10]]
Example (MakeQuantProducts)

Test code to ensure we make quant products correctly. 

var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

datasetsbucket := "datasets-bucket"
usersbucket := "users-bucket"

mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(usersbucket), Key: aws.String("UserContent/shared/mydatasetid/Quantifications/summary-myjobid.json"),
	},
	{
		Bucket: aws.String(datasetsbucket), Key: aws.String("Datasets/mydatasetid/summary.json"),
	},
}
// Real quant summary and dataset data from S3
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`
{
    "shared": false,
    "params": {
        "pmcsCount": 2809,
        "name": "sum then quantify",
        "dataBucket": "prodstack-persistencepixlisedata4f446ecf-m36oehuca7uc",
        "datasetPath": "Datasets/069927431/dataset.bin",
        "datasetID": "069927431",
        "jobBucket": "prodstack-persistencepiquantjobs65c7175e-12qccz2o7aimo",
        "detectorConfig": "PIXL/PiquantConfigs/v6",
        "elements": [
            "Ca",
            "Ti",
            "Fe",
            "Al"
        ],
        "parameters": "",
        "runTimeSec": 60,
        "coresPerNode": 4,
        "startUnixTime": 1630629236,
        "creator": {
            "name": "peternemere",
            "user_id": "5de45d85ca40070f421a3a34",
            "email": "peternemere@gmail.com"
        },
        "roiID": "",
        "elementSetID": "",
        "piquantVersion": "registry.github.com/pixlise/piquant/runner:3.2.8-ALPHA",
        "quantMode": "CombinedBulk",
        "comments": "",
        "roiIDs": [
            "scxb9y7xk027mcqq",
            "ee4e94n3fsu3lcqq",
            "4vvypcnk1xtd4sn2",
            "w28i1d4imca1o24e"
        ]
    },
    "elements": [
        "CaO",
        "TiO2",
        "FeO-T",
        "Al2O3"
    ],
    "jobId": "x80xad4l7k6vzalf",
    "status": "complete",
    "message": "Nodes ran: 1",
    "endUnixTime": 1630629323,
    "outputFilePath": "UserContent/5de45d85ca40070f421a3a34/069927431/Quantifications",
    "piquantLogList": [
        "node00001_piquant.log",
        "node00001_stdout.log"
    ]
}
`))),
	},
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`
{
 "dataset_id": "069927431",
 "group": "PIXL-FM",
 "drive_id": 0,
 "site_id": 7,
 "target_id": "?",
 "site": "",
 "target": "",
 "title": "Bellegarde",
 "sol": "0187",
 "rtt": 69927431,
 "sclk": 683483744,
 "context_image": "PCW_0187_0683484439_000RCM_N00700000699274310005075J04.png",
 "location_count": 2816,
 "data_file_size": 17623427,
 "context_images": 21,
 "normal_spectra": 5618,
 "dwell_spectra": 20,
 "bulk_spectra": 2,
 "max_spectra": 2,
 "pseudo_intensities": 2809,
 "detector_config": "PIXL",
 "create_unixtime_sec": 1642519029
}
`))),
	},
}
datasetid := "mydatasetid"
jobid := "myjobid"

fs := fileaccess.MakeS3Access(&mockS3)
ocsproducts, err := makeQuantProducts(fs, usersbucket, datasetsbucket, datasetid, jobid, 1)
if err != nil {
	fmt.Printf("%v", err)
}
ocsjson, err := json.MarshalIndent(ocsproducts, "", "  ")
if err != nil {
	fmt.Printf("%v", err)
}
fmt.Printf("%v", string(ocsjson))

Output:

{
  "OcsPath": "/ods/surface/sol/00187/soas/rdr/pixl/PQA",
  "SourceBucket": "users-bucket",
  "SourcePrefix": "Publish/Staging",
  "DatasetID": "mydatasetid",
  "JobID": "myjobid",
  "PqrFileName": "PES_0187_0683484439_000PQR_N00700000699274310005075J01.CSV",
  "PqrMetaFileName": "PES_0187_0683484439_000PQR_N00700000699274310005075J01.CSV.MET",
  "PqpFileName": "PES_0187_0683484439_000PQP_N00700000699274310005075J01.JSON",
  "PqpMetaFileName": "PES_0187_0683484439_000PQP_N00700000699274310005075J01.JSON.MET"
}
Example (MakeQuantedLocation) 
// Should just filter indexes that are valid
fmt.Println(makeQuantedLocation([]string{"Ca_%", "PMC", "Ti_%", "RTT", "filename", "Ca_int"}, []string{"1.11111", "2", "3.1415962", "5", "FileA.msa", "6"}, map[int]bool{1: true, 3: true, 4: true}))

Output:

{2 5 0 FileA.msa [1.11111 3.1415962 6]} <nil>
Example (MakeROIPMCListFileContents_AB) 
fmt.Println(makeROIPMCListFileContents(testROIs, "5x11dataset.bin", false, false, map[int32]bool{}))

Output:

5x11dataset.bin
roi1-id:7|Normal|A,15|Normal|A,388|Normal|A
roi1-id:7|Normal|B,15|Normal|B,388|Normal|B
roi2-id:7|Normal|A,450|Normal|A
roi2-id:7|Normal|B,450|Normal|B
 <nil>
Example (MakeROIPMCListFileContents_AB_Dwells) 
fmt.Println(makeROIPMCListFileContents(testROIs, "5x11dataset.bin", false, true, map[int32]bool{15: true}))

Output:

5x11dataset.bin
roi1-id:7|Normal|A,15|Normal|A,15|Dwell|A,388|Normal|A
roi1-id:7|Normal|B,15|Normal|B,15|Dwell|B,388|Normal|B
roi2-id:7|Normal|A,450|Normal|A
roi2-id:7|Normal|B,450|Normal|B
 <nil>
Example (MakeROIPMCListFileContents_Combined) 
fmt.Println(makeROIPMCListFileContents(testROIs, "5x11dataset.bin", true, false, map[int32]bool{}))

Output:

5x11dataset.bin
roi1-id:7|Normal|A,7|Normal|B,15|Normal|A,15|Normal|B,388|Normal|A,388|Normal|B
roi2-id:7|Normal|A,7|Normal|B,450|Normal|A,450|Normal|B
 <nil>
Example (MakeROIPMCListFileContents_Combined_Dwells) 
fmt.Println(makeROIPMCListFileContents(testROIs, "5x11dataset.bin", true, true, map[int32]bool{15: true}))

Output:

5x11dataset.bin
roi1-id:7|Normal|A,7|Normal|B,15|Normal|A,15|Normal|B,15|Dwell|A,15|Dwell|B,388|Normal|A,388|Normal|B
roi2-id:7|Normal|A,7|Normal|B,450|Normal|A,450|Normal|B
 <nil>
Example (MatchPMCsWithDataset) 
l := &logger.StdOutLogger{}
data := csvData{[]string{"X", "Y", "Z", "filename", "Ca_%"}, [][]string{[]string{"1", "0.40", "0", "Roastt_Laguna_Salinas_28kV_230uA_03_03_2020_111.msa", "4.5"}}}

err := matchPMCsWithDataset(&data, "non-existant-file.bin", true, l)
if err.Error() == "open non-existant-file.bin: no such file or directory" || err.Error() == "open non-existant-file.bin: The system cannot find the file specified." {
	fmt.Println("open non-existant-file.bin: Failed as expected")
}

fmt.Printf("%v, header[%v]=%v, data[%v]=%v\n", matchPMCsWithDataset(&data, "./testdata/LagunaSalinasdataset.bin", true, l), len(data.header)-1, data.header[5], len(data.data[0])-1, data.data[0][5])

data = csvData{[]string{"X", "Y", "Z", "filename", "Ca_%"}, [][]string{[]string{"1", "930.40", "0", "Roastt_Laguna_Salinas_28kV_230uA_03_03_2020_111.msa", "4.5"}}}
fmt.Println(matchPMCsWithDataset(&data, "./testdata/LagunaSalinasdataset.bin", true, l))

data = csvData{[]string{"X", "Y", "Z", "filename", "Ca_%"}, [][]string{[]string{"1", "0.40", "0", "Roastt_Laguna_Salinas_28kV_230uA_03_03_2020_116.msa", "4.5"}}}
fmt.Printf("%v, header[%v]=%v, data[%v]=%v\n", matchPMCsWithDataset(&data, "./testdata/LagunaSalinasdataset.bin", false, l), len(data.header)-1, data.header[5], len(data.data[0])-1, data.data[0][5])

Output:

open non-existant-file.bin: Failed as expected
<nil>, header[5]=PMC, data[5]=111
matchPMCsWithDataset Failed to match 1.00,930.40,0.00 to a PMC in dataset file
<nil>, header[5]=PMC, data[5]=116
Example (ParseFloatColumnValue) 
fVal, err := parseFloatColumnValue("3.1415926")
fmt.Printf("%v|%v\n", fVal, err)

fVal, err = parseFloatColumnValue("-3.15")
fmt.Printf("%v|%v\n", fVal, err)

fVal, err = parseFloatColumnValue("1.234e02")
fmt.Printf("%v|%v\n", fVal, err)

fVal, err = parseFloatColumnValue("")
fmt.Printf("%v|%v\n", fVal, err)

fVal, err = parseFloatColumnValue("nan")
fmt.Printf("%v|%v\n", fVal, err)

fVal, err = parseFloatColumnValue("-nan")
fmt.Printf("%v|%v\n", fVal, err)

Output:

3.1415925|<nil>
-3.15|<nil>
123.4|<nil>
0|strconv.ParseFloat: parsing "": invalid syntax
NaN|<nil>
NaN|<nil>
Example (ProcessQuantROIsToPMCs_Combined_CSVRowCountROICountMismatch) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, RTT, filename
15, 5.1, 400, 7890, Normal_A_roi1-id
7, 6.1, 405, 7800, Normal_A_roi1-id
12, 6.7, 407, 7700, Normal_A_roi1-id
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs)

fmt.Printf("%v\n", err)
fmt.Println(outputCSV)

Output:

PMC 12 in CSV: JobData/abc123/output/node001.pmcs_result.csv doesn't exist in ROI: roi1
Example (ProcessQuantROIsToPMCs_Combined_DownloadError) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	nil,
}

fs := fileaccess.MakeS3Access(&mockS3)
outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs)

fmt.Printf("%v\n", err)
fmt.Println(outputCSV)

Output:

Failed to read map CSV: JobData/abc123/output/node001.pmcs_result.csv
Example (ProcessQuantROIsToPMCs_Combined_InvalidPMC) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, filename, RTT
15, 5.1, 400, Normal_A_roi1-id, 7890
Qwerty, 6.1, 405, Normal_A_roi1-id, 7800
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs)

fmt.Printf("%v\n", err)
fmt.Println(outputCSV)

Output:

Failed to process map CSV: JobData/abc123/output/node001.pmcs_result.csv, invalid PMC Qwerty at line 4
Example (ProcessQuantROIsToPMCs_Combined_NoFileNameCol) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, RTT
15, 5.1, 400, 7890
7, 6.1, 405, 7800
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs)

fmt.Printf("%v\n", err)
fmt.Println(outputCSV)

Output:

Map csv: JobData/abc123/output/node001.pmcs_result.csv, does not contain a filename column (used to match up ROIs)
Example (ProcessQuantROIsToPMCs_Combined_OK) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, filename, CaO_int, RTT
15, 5.1, Normal_A_roi1-id, 400, 7890
7, 6.1, Normal_B_roi2-id, 405, 7800
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", true, testROIs)

fmt.Printf("%v\n", err)
fmt.Println(outputCSV)

Output:

<nil>
The custom header
PMC, CaO_%, filename, CaO_int, RTT
7, 5.1, Normal_A_roi1-id, 400, 7890
15, 5.1, Normal_A_roi1-id, 400, 7890
388, 5.1, Normal_A_roi1-id, 400, 7890
7, 6.1, Normal_B_roi2-id, 405, 7800
450, 6.1, Normal_B_roi2-id, 405, 7800
Example (ProcessQuantROIsToPMCs_SeparateAB_InvalidFileName) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, filename, RTT
15, 5.1, 400, Normal_A_roi1-id, 7890
15, 5.2, 401, Normal_B, 7890
7, 6.1, 405, Normal_A, 7800
7, 6.2, 406, Normal_B, 7800
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", false, testROIs)

fmt.Printf("%v\n", err)
fmt.Println(outputCSV)

Output:

Invalid file name read: Normal_B from map CSV: JobData/abc123/output/node001.pmcs_result.csv, line 4
Example (ProcessQuantROIsToPMCs_SeparateAB_OK) 
var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

const jobsBucket = "jobs-bucket"

// Some of our files are empty, not there, have content
// and they're meant to end up combined into one response...
mockS3.ExpGetObjectInput = []s3.GetObjectInput{
	{
		Bucket: aws.String(jobsBucket), Key: aws.String("JobData/abc123/output/node001.pmcs_result.csv"),
	},
}
mockS3.QueuedGetObjectOutput = []*s3.GetObjectOutput{
	{
		Body: ioutil.NopCloser(bytes.NewReader([]byte(`Header row1
PMC, CaO_%, CaO_int, filename, RTT
15, 5.1, 400, Normal_A_roi1-id, 7890
15, 5.2, 401, Normal_B_roi1-id, 7890
7, 6.1, 405, Normal_A_roi2-id, 7800
7, 6.2, 406, Normal_B_roi2-id, 7800
`))),
	},
}

fs := fileaccess.MakeS3Access(&mockS3)
outputCSV, err := processQuantROIsToPMCs(fs, jobsBucket, "JobData/abc123", "The custom header", "node001.pmcs", false, testROIs)

fmt.Printf("%v\n", err)
fmt.Println(outputCSV)

Output:

<nil>
The custom header
PMC, CaO_%, CaO_int, filename, RTT
7, 5.1, 400, Normal_A_roi1-id, 7890
7, 5.2, 401, Normal_B_roi1-id, 7890
15, 5.1, 400, Normal_A_roi1-id, 7890
15, 5.2, 401, Normal_B_roi1-id, 7890
388, 5.1, 400, Normal_A_roi1-id, 7890
388, 5.2, 401, Normal_B_roi1-id, 7890
7, 6.1, 405, Normal_A_roi2-id, 7800
7, 6.2, 406, Normal_B_roi2-id, 7800
450, 6.1, 405, Normal_A_roi2-id, 7800
450, 6.2, 406, Normal_B_roi2-id, 7800
Example (ReadCSV) 
csv := `something header
more header
col 1,"col, 2",  col_3
"value one",123, 456
value two,444,555
`
d, err := readCSV(csv, 2)
fmt.Printf("%v|%v", d, err)

Output:

{[col 1 col, 2 col_3] [[value one 123 456] [value two 444 555]]}|<nil>
Example (StageQuant)

Stage the quantification ready for uploading to OCS 

var mockS3 awsutil.MockS3Client
defer mockS3.FinishTest()

datasetsbucket := "datasets-bucket"

mockS3.ExpCopyObjectInput = []s3.CopyObjectInput{
	{
		Bucket:     aws.String(datasetsbucket),
		Key:        aws.String("Publish/Staging/mydatasetid/myjobid.csv"),
		CopySource: aws.String(datasetsbucket + "/UserContent/shared/mydatasetid/Quantifications/myjobid.csv"),
	},
	{
		Bucket:     aws.String(datasetsbucket),
		Key:        aws.String("Publish/Staging/mydatasetid/summary-myjobid.json"),
		CopySource: aws.String(datasetsbucket + "/UserContent/shared/mydatasetid/Quantifications/summary-myjobid.json"),
	},
}
mockS3.QueuedCopyObjectOutput = []*s3.CopyObjectOutput{
	{},
	{},
}
datasetid := "mydatasetid"
jobid := "myjobid"
products := ProductSet{
	OcsPath:         "",
	SourceBucket:    "",
	SourcePrefix:    "",
	DatasetID:       "",
	JobID:           "",
	PqrFileName:     "myjobid.csv",
	PqrMetaFileName: "myjobid.csv.met",
	PqpFileName:     "summary-myjobid.json",
	PqpMetaFileName: "summary-myjobid.json.met",
}
fs := fileaccess.MakeS3Access(&mockS3)
err := stageQuant(fs, datasetsbucket, datasetid, jobid, products)
if err != nil {
	fmt.Printf("%v", err)
}

Output:
Example (ValidateParameters) 
fmt.Printf("%v\n", validateParameters("-b,0,50,2,10 -f"))
fmt.Printf("%v\n", validateParameters("-b,0,50,2,10.55 -o \"filename.whatever\" -f -Fe,1"))
fmt.Printf("%v\n", validateParameters("-b,0,50,2,10;ls -al;echo -f"))
fmt.Printf("%v\n", validateParameters("-b,0,50,2,10&&rm -rf ~/; -f"))

Output:

<nil>
<nil>
Invalid parameters passed: -b,0,50,2,10;ls -al;echo -f
Invalid parameters passed: -b,0,50,2,10&&rm -rf ~/; -f

Index

Examples

Constants

View Source 
const (
	JobStarting         JobStatusValue = "starting"
	JobPreparingNodes                  = "preparing_nodes"
	JobNodesRunning                    = "nodes_running"
	JobGatheringResults                = "gathering_results"
	JobComplete                        = "complete"
	JobError                           = "error"
)

Valid job status strings

View Source 
const JobParamsFileName = "params.json"

JobParamsFileName - File name of job params file

View Source 
const OcsPath string = "/ods/surface/sol/XXXXX/soas/rdr/pixl/PQA"

OcsPath directs publishing to ODS

View Source 
const OcsStagingPath string = "Publish/Staging"

OcsStagingPath path in s3 bucket where files are staged before publishing to ODS

View Source 
const PublicationsPath string = "Publish/publications.json"

PublicationsPath path in s3 bucket where the list of publication records are kept

View Source 
const QuantModeABManualUpload = "ABManual"
View Source 
const QuantModeABMultiQuant = "ABMultiQuant"
View Source 
const QuantModeCombinedManualUpload = "ABManual"
View Source 
const QuantModeCombinedMultiQuant = "CombinedMultiQuant"

Variables

This section is empty.

Functions

func CheckQuantificationNameExists 

func CheckQuantificationNameExists(name string, datasetID string, userID string, svcs *services.APIServices) bool

func ConvertQuantificationCSV 

func ConvertQuantificationCSV(logger logger.ILogger, data string, expectMetaColumns []string, matchPMCDatasetFileName string, matchPMCByCoord bool, detectorIDOverride string, detectorDuplicateAB bool) ([]byte, []string, error)

ConvertQuantificationCSV - converts from incoming string CSV data to serialised binary data Returns the serialised quantification bytes and the elements that were quantified

func CreateJob 

func CreateJob(svcs *services.APIServices, createParams JobCreateParams, wg *sync.WaitGroup) (string, error)

CreateJob - creates a new quantification job

func GetBlessedQuantFile 

func GetBlessedQuantFile(svcs *services.APIServices, datasetID string) (BlessFile, *BlessFileItem, string, error)

Downloads & parses the blessed quants file. Returns: - the parsed contents - the blessed quant job info (BlessItem) - the path (in case we want to update the same file) - error or nil

func GetQuantColumnIndex 

func GetQuantColumnIndex(quant *protos.Quantification, column string) int32

GetQuantColumnIndex - returns index of column in quantification or -1 if not found

func GetQuantification 

func GetQuantification(svcs *services.APIServices, s3Path string) (*protos.Quantification, error)

func GetWeightPercentColumnsInQuant 

func GetWeightPercentColumnsInQuant(quant *protos.Quantification) []string

GetWeightPercentColumnsInQuant - returns weight % columns, ones ending in _%

func ImportQuantCSV 

func ImportQuantCSV(svcs *services.APIServices, datasetID string, importUser pixlUser.UserInfo, csvBody string, csvOrigin string, idPrefix string, quantName string, quantModeEnum string, comments string) (string, error)

func PublishQuant 

func PublishQuant(fs fileaccess.FileAccess, config PublisherConfig, creator pixlUser.UserInfo, log logger.ILogger, datasetID, jobID string, notifications notifications.NotificationManager) error

func ReadQuantificationFile 

func ReadQuantificationFile(path string) (*protos.Quantification, error)

func ShareQuantification 

func ShareQuantification(svcs *services.APIServices, userID string, datasetID string, jobID string) error

Types

type BlessFile 

type BlessFile struct {
	History []BlessFileItem `json:"history"`
}

type BlessFileItem 

type BlessFileItem struct {
	Version      int    `json:"version"`
	BlessUnixSec int64  `json:"blessedAt"`
	UserID       string `json:"userId"`
	UserName     string `json:"userName"`
	JobID        string `json:"jobId"`
}

type DetectorCode 

type DetectorCode string

DetectorCode Indicates the detector used in a Quantification; A/B or combined 

const (
	Combined DetectorCode = "C" // Combined uses the combined detector for quantification
	Separate              = "S" // Separate uses the separate A/B detectors for quantification
)

type JobCreateParams 

type JobCreateParams struct {
	Name           string            `json:"name"`
	DatasetPath    string            `json:"datasetPath"`
	PMCs           []int32           `json:"pmcs"`
	Elements       []string          `json:"elements"`
	DetectorConfig string            `json:"detectorconfig"`
	Parameters     string            `json:"parameters"`
	RunTimeSec     int32             `json:"runtimesec"`
	RoiID          string            `json:"roiID"` // There is now a list of ROI IDs that can be provided too. More relevant with the QuantMode *Bulk options
	ElementSetID   string            `json:"elementSetID"`
	DatasetID      string            `json:"dataset_id"`
	Creator        pixlUser.UserInfo `json:"creator"`
	QuantMode      string            `json:"quantMode"`
	RoiIDs         []string          `json:"roiIDs"` // If QuantMode = *Bulk, this is used, pmcs is ignored.
	IncludeDwells  bool              `json:"includeDwells,omitempty"`
	Command        string            `json:"command,omitempty"`
}

JobCreateParams - Parameters to CreateJob()

type JobStartingParameters 

type JobStartingParameters struct {
	Name              string            `json:"name"`
	DataBucket        string            `json:"dataBucket"`
	DatasetPath       string            `json:"datasetPath"`
	DatasetID         string            `json:"datasetID"`
	PiquantJobsBucket string            `json:"jobBucket"`
	DetectorConfig    string            `json:"detectorConfig"`
	Elements          []string          `json:"elements"`
	Parameters        string            `json:"parameters"`
	RunTimeSec        int32             `json:"runTimeSec"`
	CoresPerNode      int32             `json:"coresPerNode"`
	StartUnixTime     int64             `json:"startUnixTime"`
	Creator           pixlUser.UserInfo `json:"creator"`
	RoiID             string            `json:"roiID"`
	ElementSetID      string            `json:"elementSetID"`
	PIQUANTVersion    string            `json:"piquantVersion"`
	QuantMode         string            `json:"quantMode"`
	Comments          string            `json:"comments"`
	RoiIDs            []string          `json:"roiIDs"`
	IncludeDwells     bool              `json:"includeDwells,omitempty"`
	Command           string            `json:"command,omitempty"`
}

JobStartingParameters - parameters to start the job, saved in S3 as JobParamsFileName for the job

type JobStartingParametersWithPMCCount 

type JobStartingParametersWithPMCCount struct {
	PMCCount int32 `json:"pmcsCount"`
	*JobStartingParameters
}

JobStartingParametersWithPMCCount - summary version of JobStartingParametersWithPMCs, only storing PMC count

func MakeJobStartingParametersWithPMCCount 

func MakeJobStartingParametersWithPMCCount(params JobStartingParametersWithPMCs) JobStartingParametersWithPMCCount

MakeJobStartingParametersWithPMCCount - Converting full to summary version of struct

type JobStartingParametersWithPMCs 

type JobStartingParametersWithPMCs struct {
	PMCs []int32 `json:"pmcs"`
	*JobStartingParameters
}

JobStartingParametersWithPMCs - When saving file with name: JobParamsFileName, we save all PMCs, but in further references to the job parameters, eg in job list summary (filepaths.JobSummarySuffix), we only store the PMC count

type JobStatus 

type JobStatus struct {
	JobID          string         `json:"jobId"`
	Status         JobStatusValue `json:"status"`
	Message        string         `json:"message"`
	EndUnixTime    int64          `json:"endUnixTime"`
	OutputFilePath string         `json:"outputFilePath"`
	PiquantLogList []string       `json:"piquantLogList"`
}

JobStatus - job status that gets saved to S3 as <job-id>JobStatusSuffix as it progresses

type JobStatusValue 

type JobStatusValue string

JobStatusValue - the type for our job status field

type JobSummaryItem 

type JobSummaryItem struct {
	Shared   bool                              `json:"shared"`
	Params   JobStartingParametersWithPMCCount `json:"params"`
	Elements []string                          `json:"elements"`
	*JobStatus
}

JobSummaryItem all metadata stored for an individual job/quant file (even after it was generated)

func GetJobSummary 

func GetJobSummary(fs fileaccess.FileAccess, bucket string, userID string, datasetID string, jobID string) (JobSummaryItem, error)

GetJobSummary - Get the summary information from a quant job json file in S3

func ListQuantJobsForDataset 

func ListQuantJobsForDataset(svcs *services.APIServices, userID string, datasetID string) ([]JobSummaryItem, error)

Downloads job summary file with path: s3://PiquantJobsBucket/RootJobSummaries/<dataset-id>JobSummarySuffix Returns only jobs created by userID that are NOT complete (still in progress/failed)

func SetMissingSummaryFields 

func SetMissingSummaryFields(summary JobSummaryItem) JobSummaryItem

SetMissingSummaryFields - ensure the fields all exist over time.

type JobSummaryMap 

type JobSummaryMap map[string]JobSummaryItem

JobSummaryMap - saved by job updater lambda function, read by quant listing handler

type OcsMetaData 

type OcsMetaData struct {
	Description string `json:"description"`
}

OcsMetaData metadata file that provides information to OCS about published Datasets

type PiquantParams 

type PiquantParams struct {
	RunTimeEnv        string   `json:"runtimeEnv"`
	JobID             string   `json:"jobId"`
	JobsPath          string   `json:"jobsPath"`
	DatasetPath       string   `json:"datasetPath"`
	DetectorConfig    string   `json:"detectorConfig"`
	Elements          []string `json:"elements"`
	Parameters        string   `json:"parameters"`
	DatasetsBucket    string   `json:"datasetsBucket"`
	ConfigBucket      string   `json:"configBucket"`
	PiquantJobsBucket string   `json:"jobBucket"`
	QuantName         string   `json:"name"`
	PMCListName       string   `json:"pmcListName"`
	Command           string   `json:"command"`
}

PiquantParams - Parameters for running piquant, as generated by PIXLISE API

type ProductSet 

type ProductSet struct {
	OcsPath         string
	SourceBucket    string
	SourcePrefix    string
	DatasetID       string
	JobID           string
	PqrFileName     string
	PqrMetaFileName string
	PqpFileName     string
	PqpMetaFileName string
}

type ProductType 

type ProductType string

ProductType Indicates the Product Type to publish to OCS 

const (
	PiQuantResults ProductType = "PQR" // PiQuantResults PQR Products contain the PiQuant Quantification Results
	PiQuantParams  ProductType = "PQP" // PiQuantParams PQP Products contain the PiQuant Runtime Parameters

)

type Publication 

type Publication struct {
	Version         int       `json:"version"`
	PublicationTime time.Time `json:"timestamp"`
	Publisher       string    `json:"publisher"`
}

Publication keeps details about a particular publication

type PublicationRecord 

type PublicationRecord struct {
	DatasetID    string        `json:"dataset-id"`
	JobID        string        `json:"job-id"`
	Publications []Publication `json:"publications"`
}

PublicationRecord keeps tracks all the publications for a quantification

type Publications 

type Publications struct {
	Datasets []PublicationRecord `json:"datasets"`
}

Publications keeps a list of publications for all Datasets

type PublisherConfig 

type PublisherConfig struct {
	KubernetesLocation      string
	QuantDestinationPackage string
	QuantObjectType         string
	PosterImage             string
	DatasetsBucket          string
	EnvironmentName         string
	Kubeconfig              string
	UsersBucket             string
}

type ROIWithPMCs 

type ROIWithPMCs struct {
	PMCs []int
	ID   string
	*roiModel.ROISavedItem
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.