csvutil

func FilterCSVFile ¶

func FilterCSVFile(inPath, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error

func MergeFilterCSVFiles ¶

func MergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error

MergeFilterCSVFiles can merge and filter multiple CSV files. It expects row definitions to be the same across all input files.

func MergeFilterCSVFilesToJSONL ¶

func MergeFilterCSVFilesToJSONL(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error

type JsonRecordsInfo struct {
	Meta    JsonRecordsInfoMeta `json:"meta"`
	Records []map[string]string `json:"records"`
}

type JsonRecordsInfoMeta struct {
	Count int `json:"count"`
}

func ReadMergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) (table.DocumentsSet, error) {
	//data := JsonRecordsInfo{Records: []map[string]string{}}
	data := table.NewDocumentsSet()

	for _, inPath := range inPaths {
		reader, inFile, err := NewReader(inPath, inComma, inStripBom)
		if err != nil {
			return data, err
		}

		csvHeader := CSVHeader{}
		j := -1

		for {
			line, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				return data, err
			}
			j += 1

			if j == 0 {
				csvHeader.Columns = line
				continue
			}
			match, err := csvHeader.RecordMatch(line, andFilter)
			if err != nil {
				return data, err
			}
			if !match {
				continue
			}

			mss := csvHeader.RecordToMSS(line)
			data.Documents = append(data.Documents, mss)
		}
		err = inFile.Close()
		if err != nil {
			return data, err
		}
	}
	data.Inflate()
	return data, nil
}

func MergeFilterCSVFilesToJSON(inPaths []string, outPath string, inComma rune, inStripBom bool, perm os.FileMode, andFilter map[string]stringsutil.MatchInfo) error {
	data, err := ReadMergeFilterCSVFiles(inPaths, outPath, inComma, inStripBom, andFilter)
	if err != nil {
		return err
	}
	bytes, err := jsonutil.MarshalSimple(data, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(outPath, bytes, perm)
}

func NewReader ¶

func NewReader(path string, comma rune, stripBom bool) (*csv.Reader, *os.File, error)

NewReader will create a csv.Reader and optionally strip off the byte order mark (BOM) if requested. Close file reader with `defer f.Close()`.

func NewWriterFile ¶

func NewWriterFile(filename string) (*csv.Writer, *os.File, error)

func ReadCSVFileSingleColumnValuesString ¶

func ReadCSVFileSingleColumnValuesString(filename string, sep rune, stripBOM, hasHeader, trimSpace bool, colIdx uint, condenseUniqueSort bool) ([]string, error)

func ReadCSVFilesSingleColumnValuesString ¶

func ReadCSVFilesSingleColumnValuesString(files []string, sep rune, stripBOM, hasHeader, trimSpace bool, colIdx uint, condenseUniqueSort bool) ([]string, error)

func WriteCSVFiltered ¶

func WriteCSVFiltered(reader *csv.Reader, writer *csv.Writer, andFilter map[string]stringsutil.MatchInfo, writeHeader bool) error

WriteCSVFiltered filters an existing CSV and writes the matching lines to a *csv.Writer.