ingester

package
v0.11.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 15, 2020 License: AGPL-3.0, AGPL-3.0 Imports: 35 Imported by: 0

README

ingester

file ingester for gradex

Documentation

Index

Constants

View Source
const (
	QuestionReady  = "questionReady"
	QuestionSent   = "questionSent"
	MarkerReady    = "markerReady"
	MarkerSent     = "markerSent"
	ModeratorReady = "moderatorReady"
	ModeratorSent  = "moderatorSent"
	CheckerReady   = "checkerReady"
	CheckerSent    = "checkerSent"
	ReMarkerReady  = "remarkerReady"
	ReMarkerSent   = "remarkerSent"
	ReCheckerReady = "recheckerReady"
	ReCheckerSent  = "recheckerSent"

	PageBad           = pageBad
	MarkerProcessed   = markerProcessed
	ModeratorActive   = moderatorActive
	ModeratorInactive = moderatorInactive

	//External stages
	Checked = checked
)

Variables

View Source
var (
	ExamStage = []string{
		config,
		pageBad,
		acceptedReceipts,
		acceptedPapers,
		tempImages,
		tempPages,
		anonPapers,
		questionImages,
		questionPages,
		questionReady,
		questionSent,
		questionBack,
		questionSplit,
		markerReady,
		markerSent,
		markerBack,
		markerFlattened,
		markerProcessed,
		moderatorInactive,
		moderatorActive,
		moderatorReady,
		moderatorSent,
		moderatorBack,
		moderatorFlattened,
		moderatorProcessed,
		enterInactive,
		enterActive,
		enterReady,
		enterSent,
		enterBack,
		enterFlattened,
		enterProcessed,
		checkerReady,
		checkerSent,
		checkerBack,
		checkerFlattened,
		checkerProcessed,
		finalCover,
		reMarkerInactive,
		reMarkerActive,
		reMarkerReady,
		reMarkerSent,
		reMarkerBack,
		reMarkerFlattened,
		reMarkerProcessed,
		reModeratorInactive,
		reModeratorActive,
		reModeratorReady,
		reModeratorSent,
		reModeratorBack,
		reModeratorFlattened,
		reModeratorProcessed,
		reCheckerReady,
		reCheckerSent,
		reCheckerBack,
		reCheckerFlattened,
		reCheckerProcessed,
		finalPapers,
		reports,
	}
)

Functions

func BareFile

func BareFile(name string) string

func BaseList

func BaseList(paths []string) []string

func CommentsToString added in v0.5.0

func CommentsToString(comments []comment.Comment) string

func ConvertPDFToJPEGs

func ConvertPDFToJPEGs(pdfPath string, jpegPath string, outputFile string) error

func Copy

func Copy(source, destination string) error

func CopyIsComplete

func CopyIsComplete(source, dest []string) bool

func CopyOverWrite added in v0.5.0

func CopyOverWrite(source, destination string) error

func CountPDFInDir added in v0.7.2

func CountPDFInDir(dir string) (int, error)

func CountPages

func CountPages(inputPath string) (int, error)

func CropToQuestion

func CropToQuestion(inputPath, outputPath string) error

func DoOneCoverPage added in v0.6.0

func DoOneCoverPage(ct CoverPageTask, logger *zerolog.Logger) error

func DoOneFinalCoverPage added in v0.9.0

func DoOneFinalCoverPage(ct CoverPageTask, logger *zerolog.Logger) error

func EnsureDir

func EnsureDir(dirName string) error

func EnsureDirAll

func EnsureDirAll(dirName string) error

func GetAnonymousFileName

func GetAnonymousFileName(course, anonymousIdentity string) string

func GetAnonymousFromPath added in v0.8.0

func GetAnonymousFromPath(path string) string

looks for the word between - and .pdf (case insensitive) at the end of the string

func GetAnonymousFromPathBasic added in v0.8.0

func GetAnonymousFromPathBasic(path string) string

looks for the word between - and .pdf (case insensitive) at the end of the string

func GetDone added in v0.7.2

func GetDone(path string) bool

func GetField added in v0.5.0

func GetField(fields []pagedata.Field, key string) string

func GetFileList

func GetFileList(dir string) ([]string, error)

func GetFileListThisDir

func GetFileListThisDir(dir string) ([]string, error)

func GetPDFPath

func GetPDFPath(filename, directory string) (string, error)

when we read the Learn receipt, we might get a suffix for a word doc etc so find the pdf file in the target directory with the same base prefix name but possibly variable capitalisation of the suffix (handmade file!)

func GetPageSummaryMap added in v0.5.0

func GetPageSummaryMap(pdMap map[int]pagedata.PageData) (map[int]PageReport, error)

func GetPageSummaryMapFromFile added in v0.5.0

func GetPageSummaryMapFromFile(path string) (map[int]PageReport, error)

func GetRequiredQuestions added in v0.7.0

func GetRequiredQuestions(qfile string) ([]string, error)

func GetShortActorName added in v0.5.0

func GetShortActorName(name string) string

some external libraries use this, like pagedata

func GetShortLearnDate

func GetShortLearnDate(sub parselearn.Submission) (string, error)

func GetSubDirList added in v0.5.0

func GetSubDirList(dir string) ([]string, error)

func IsArchive

func IsArchive(path string) bool

func IsCSV

func IsCSV(path string) bool

func IsEmptyDir added in v0.5.0

func IsEmptyDir(name string) (bool, error)

https://rosettacode.org/wiki/Empty_directory#Go

func IsPDF

func IsPDF(path string) bool

need to be case insensitive

func IsTXT

func IsTXT(path string) bool

func IsZIP

func IsZIP(path string) bool

func ItemExists

func ItemExists(sliceType interface{}, item interface{}) bool

Mod from array to slice, from https://www.golangprograms.com/golang-check-if-array-element-exists.html

func SetDone added in v0.7.2

func SetDone(path string, logger *zerolog.Logger)

func ValidStageForExport added in v0.5.0

func ValidStageForExport(stage string) bool

func ValidStageForProcessedPapers added in v0.5.0

func ValidStageForProcessedPapers(stage string) bool

initial sanity check on stage that has been specified also used by merge "half" of the process (see merge.go)

func WritePageReportsToCSV added in v0.6.1

func WritePageReportsToCSV(reports []PageReport, outputPath string) error

Types

type CoverPageCommand added in v0.6.0

type CoverPageCommand struct {
	Questions      []string
	FromPath       string
	ToPath         string
	ConfigPath     string
	ExamName       string
	TemplatePath   string
	SpreadName     string
	ProcessDetail  pagedata.ProcessDetail
	PathDecoration string
}

type CoverPageTask added in v0.6.0

type CoverPageTask struct {
	Command CoverPageCommand
	Path    string
}

type FlattenTask

type FlattenTask struct {
	InputPath   string
	PageCount   int
	PageDataMap map[int]pagedata.PageData
	OutputPath  string
	PreparedFor string
	ToDo        string
}

type Ingester

type Ingester struct {
	Redo                  bool
	UseFullAssignmentName bool

	SkipQuestionFile bool //TODO revert to private, probably
	// contains filtered or unexported fields
}

func New

func New(path string, msgCh chan chmsg.MessageInfo, logger *zerolog.Logger) (*Ingester, error)

func (*Ingester) AddCheckBar

func (g *Ingester) AddCheckBar(exam string, checker string) error

func (*Ingester) AddCheckCoverBar added in v0.6.0

func (g *Ingester) AddCheckCoverBar(exam string, checker string) error

Add a cover page summarising the marking done so far

func (*Ingester) AddEnterActiveBar added in v0.5.0

func (g *Ingester) AddEnterActiveBar(exam string, enterer string) error

func (*Ingester) AddEnterInactiveBar added in v0.5.0

func (g *Ingester) AddEnterInactiveBar(exam string) error

func (*Ingester) AddFinalCover added in v0.9.0

func (g *Ingester) AddFinalCover(exam string, checker string) error

Add a cover page summarising the finalised marking summary

func (*Ingester) AddLabelBar

func (g *Ingester) AddLabelBar(exam, labeller string) error

func (*Ingester) AddMarkBar

func (g *Ingester) AddMarkBar(exam string, marker string) error

func (*Ingester) AddMarkBarByQ

func (g *Ingester) AddMarkBarByQ(exam string, marker string) error

func (*Ingester) AddModerateActiveBar

func (g *Ingester) AddModerateActiveBar(exam string, moderator string) error

func (*Ingester) AddModerateInActiveBar

func (g *Ingester) AddModerateInActiveBar(exam string) error

func (*Ingester) BareFile

func (g *Ingester) BareFile(name string) string

func (*Ingester) BaseList

func (g *Ingester) BaseList(paths []string) []string

func (*Ingester) CheckReport added in v0.7.0

func (g *Ingester) CheckReport(exam string) error

func (*Ingester) CleanFromIngest

func (g *Ingester) CleanFromIngest() error

func (*Ingester) Copy

func (g *Ingester) Copy(source, destination string) error

func (*Ingester) CopyIfNewerThanDestination added in v0.5.0

func (g *Ingester) CopyIfNewerThanDestination(source, destination string, logger *zerolog.Logger) (bool, error)

if the source file is not newer, it's not an error we just won't move it - anything left we deal with later also, we delete "<file>.done" indicator

func (*Ingester) CopyIfNewerThanDestinationInDir added in v0.5.0

func (g *Ingester) CopyIfNewerThanDestinationInDir(source, destinationDir string, logger *zerolog.Logger) (bool, error)

returns true if moved

func (*Ingester) CopyIsComplete

func (g *Ingester) CopyIsComplete(source, dest []string) bool

func (*Ingester) CopyToDir

func (g *Ingester) CopyToDir(source, destinationDir string) error

func (*Ingester) CoverPage added in v0.6.0

func (g *Ingester) CoverPage(cp CoverPageCommand, logger *zerolog.Logger) error

func (*Ingester) EnsureDir

func (g *Ingester) EnsureDir(dirName string) error

func (*Ingester) EnsureDirAll

func (g *Ingester) EnsureDirAll(dirName string) error

func (*Ingester) EnsureDirectoryStructure

func (g *Ingester) EnsureDirectoryStructure() error

This must remain idempotent so we can call it every startup

func (*Ingester) Etc

func (g *Ingester) Etc() string

func (*Ingester) Exam

func (g *Ingester) Exam() string

func (*Ingester) Export

func (g *Ingester) Export() string

func (*Ingester) ExportFiles added in v0.5.0

func (g *Ingester) ExportFiles(exam, stage, actor string) error

func (*Ingester) ExtractConf

func (g *Ingester) ExtractConf() string

func (*Ingester) FinalCoverPage added in v0.9.0

func (g *Ingester) FinalCoverPage(cp CoverPageCommand, logger *zerolog.Logger) error

func (*Ingester) FinalReport added in v0.9.0

func (g *Ingester) FinalReport(exam string) error

func (*Ingester) FlattenLayoutSVG

func (g *Ingester) FlattenLayoutSVG() string

note these rely on info contained in the instantiated ingester because they can be set on the command line

func (*Ingester) FlattenNewPapers

func (g *Ingester) FlattenNewPapers(exam string) error

func (*Ingester) FlattenOneNewPDF added in v0.5.0

func (g *Ingester) FlattenOneNewPDF(inputPath, outputPath string, pageDataMap map[int]pagedata.PageData, logger *zerolog.Logger) (int, error)

func (*Ingester) FlattenProcessedPapers added in v0.5.0

func (g *Ingester) FlattenProcessedPapers(exam, stage string) error

func (*Ingester) FlattenProcessedPapersFromDir added in v0.5.0

func (g *Ingester) FlattenProcessedPapersFromDir(exam, stage string) (string, error)

func (*Ingester) FlattenProcessedPapersToDir added in v0.5.0

func (g *Ingester) FlattenProcessedPapersToDir(exam, stage string) (string, error)

func (*Ingester) GetAnonymousFileName

func (g *Ingester) GetAnonymousFileName(course, anonymousIdentity string) string

func (*Ingester) GetEmptySubDirs added in v0.5.0

func (g *Ingester) GetEmptySubDirs(dir string) ([]string, error)

func (*Ingester) GetExamDir added in v0.5.0

func (g *Ingester) GetExamDir(exam, dir string) string

func (*Ingester) GetExamDirNamed added in v0.5.0

func (g *Ingester) GetExamDirNamed(exam, dir, name string) string

func (*Ingester) GetExamDirSub added in v0.5.0

func (g *Ingester) GetExamDirSub(exam, dir, sub string) string

note that inactive moderator back would use this function as destination := GetExamDir(exam, moderatorBack, inactive)

func (*Ingester) GetExamRoot added in v0.5.0

func (g *Ingester) GetExamRoot(exam string) string

func (*Ingester) GetExportDir added in v0.5.0

func (g *Ingester) GetExportDir(exam, dir, name string) string

func (*Ingester) GetExportDirs added in v0.5.0

func (g *Ingester) GetExportDirs(exam, stage, actor string) (string, string, string, error)

func (*Ingester) GetFileList

func (g *Ingester) GetFileList(dir string) ([]string, error)

func (*Ingester) GetFileListThisDir

func (g *Ingester) GetFileListThisDir(dir string) ([]string, error)

func (*Ingester) GetNamedTaskDecoration added in v0.5.0

func (g *Ingester) GetNamedTaskDecoration(task, name string) string

func (*Ingester) GetShortActorName added in v0.5.0

func (g *Ingester) GetShortActorName(name string) string

func (*Ingester) Identity

func (g *Ingester) Identity() string

func (*Ingester) IdentityCSV

func (g *Ingester) IdentityCSV() string

func (*Ingester) Ingest

func (g *Ingester) Ingest() string

func (*Ingester) IngestConf

func (g *Ingester) IngestConf() string

func (*Ingester) IngestTemplate

func (g *Ingester) IngestTemplate() string

func (*Ingester) IsArchive

func (g *Ingester) IsArchive(path string) bool

func (*Ingester) IsCsv

func (g *Ingester) IsCsv(path string) bool

func (*Ingester) IsPDF

func (g *Ingester) IsPDF(path string) bool

func (*Ingester) IsSameAsSelfInDir

func (g *Ingester) IsSameAsSelfInDir(source, destinationDir string) bool

func (*Ingester) IsTxt

func (g *Ingester) IsTxt(path string) bool

func (*Ingester) IsZip

func (g *Ingester) IsZip(path string) bool

func (*Ingester) ItemExists

func (g *Ingester) ItemExists(sliceType interface{}, item interface{}) bool

Mod from array to slice, from https://www.golangprograms.com/golang-check-if-array-element-exists.html

func (*Ingester) MergeOverlayOnePDF added in v0.5.0

func (g *Ingester) MergeOverlayOnePDF(mt MergeTask, logger *zerolog.Logger) (int, error)

---------------- MergeOverlayOnePDF -----------------------------------------------------

-----------------------------------------------------------------------------------------

func (*Ingester) MergeOverlayPapers added in v0.5.0

func (g *Ingester) MergeOverlayPapers(mc MergeCommand, logger *zerolog.Logger) error

we pass pointer to logger that has a processing stage string pre-prended to it so we can tell what stage overlay is being used at

func (*Ingester) MergeProcessedPapers added in v0.5.0

func (g *Ingester) MergeProcessedPapers(exam, stage string) error

func (*Ingester) MergeProcessedPapersFromDir added in v0.5.0

func (g *Ingester) MergeProcessedPapersFromDir(exam, stage string) (string, error)

func (*Ingester) MergeProcessedPapersToDir added in v0.5.0

func (g *Ingester) MergeProcessedPapersToDir(exam, stage string) (string, error)

func (*Ingester) MigrateTempImages added in v0.5.0

func (g *Ingester) MigrateTempImages(exam string, test bool) error

on update from 0.4 -0.5 temp images moved

func (*Ingester) MigrateVersionDirStruct added in v0.5.0

func (g *Ingester) MigrateVersionDirStruct(exam string, test bool) error

func (*Ingester) MoveIfNewerThanDestination

func (g *Ingester) MoveIfNewerThanDestination(source, destination string, logger *zerolog.Logger) (bool, error)

if the source file is not newer, it's not an error we just won't move it - anything left we deal with later also, we delete "<file>.done" indicator

func (*Ingester) MoveIfNewerThanDestinationInDir

func (g *Ingester) MoveIfNewerThanDestinationInDir(source, destinationDir string, logger *zerolog.Logger) (bool, error)

returns true if moved

func (*Ingester) MoveToDir

func (g *Ingester) MoveToDir(source, destinationDir string) error

func (*Ingester) OutputPath

func (g *Ingester) OutputPath(dir, inPath, decoration string) string

func (*Ingester) OverlayConf

func (g *Ingester) OverlayConf() string

func (*Ingester) OverlayLayoutSVG

func (g *Ingester) OverlayLayoutSVG() string

func (*Ingester) OverlayOnePDF

func (g *Ingester) OverlayOnePDF(ot OverlayTask, logger *zerolog.Logger) (int, error)

-----------------------------OverlayOnePDF----------------------------------------- do one file, dynamically assembling the data we need make the latest pagedata from what we get in the OverlayTask struct return the number of pages -----------------------------------------------------------------------------------

func (*Ingester) OverlayPapers

func (g *Ingester) OverlayPapers(oc OverlayCommand, logger *zerolog.Logger) error

we pass pointer to logger that has a processing stage string pre-prended to it so we can tell what stage overlay is being used at

func (*Ingester) OverlayTemplate

func (g *Ingester) OverlayTemplate() string

func (*Ingester) RemoveEmptySubDirs added in v0.5.0

func (g *Ingester) RemoveEmptySubDirs(dir string, test bool) error

func (*Ingester) ReportOnProcessedDir added in v0.5.0

func (g *Ingester) ReportOnProcessedDir(exam, dir string, showOK bool, showMark bool, reconcile bool) ([]string, error)

func (*Ingester) Root

func (g *Ingester) Root() string

func (*Ingester) SetBackgroundIsVanilla added in v0.5.0

func (g *Ingester) SetBackgroundIsVanilla(vanilla bool)

func (*Ingester) SetChangeAncestor added in v0.8.0

func (g *Ingester) SetChangeAncestor(change bool)

func (*Ingester) SetIngestTemplatePath added in v0.4.0

func (g *Ingester) SetIngestTemplatePath(path string) error

func (*Ingester) SetOpticalShrink added in v0.5.0

func (g *Ingester) SetOpticalShrink(shrink int)

func (*Ingester) SetOverlayTemplatePath added in v0.4.0

func (g *Ingester) SetOverlayTemplatePath(path string) error

func (*Ingester) SetSkipQuestionFile added in v0.7.2

func (g *Ingester) SetSkipQuestionFile(skip bool)

func (*Ingester) SetTesting

func (g *Ingester) SetTesting()

func (*Ingester) SetUseFullAssignmentName added in v0.3.0

func (g *Ingester) SetUseFullAssignmentName()

func (*Ingester) SetupConf

func (g *Ingester) SetupConf() string

func (*Ingester) SetupExamDirs added in v0.5.0

func (g *Ingester) SetupExamDirs(exam string) error

func (*Ingester) SetupGradexDirs added in v0.5.0

func (g *Ingester) SetupGradexDirs() error

func (*Ingester) SortCheck added in v0.2.1

func (g *Ingester) SortCheck(exam string) error

func (*Ingester) SortQuestions

func (g *Ingester) SortQuestions(exam string) error

func (*Ingester) SplitForEnter added in v0.5.0

func (g *Ingester) SplitForEnter(exam string) error

func (*Ingester) SplitForModeration

func (g *Ingester) SplitForModeration(exam string, minFiles int, minPercent float64) error

func (*Ingester) StageFromIngest

func (g *Ingester) StageFromIngest() error

wait for user to press an "do ingest button", then filewalk to get the paths

func (*Ingester) TempPDF

func (g *Ingester) TempPDF() string

func (*Ingester) TempTXT

func (g *Ingester) TempTXT() string

func (*Ingester) Usr

func (g *Ingester) Usr() string

func (*Ingester) ValidateNewPapers

func (g *Ingester) ValidateNewPapers() error

func (*Ingester) Var

func (g *Ingester) Var() string

type Mark added in v0.7.0

type Mark struct {
	Q string
	V string
}

func GetMarksFromCoverPage added in v0.7.0

func GetMarksFromCoverPage(path string) ([]Mark, pagedata.ItemDetail, error)

type MarkCompare added in v0.9.0

type MarkCompare struct {
	Who        string
	What       string
	When       string
	Final      []Mark
	Draft      []Mark
	DraftTotal float64
	FinalTotal float64
	Comment    string
}

type MarkSub added in v0.8.4

type MarkSub struct {
	Q    string `csv:"Q"`
	Mark string `csv:"mark"`
}

type MergeCommand added in v0.5.0

type MergeCommand struct {
	MergeFiles    []MergeFile
	ToDir         string
	Template      string
	SpreadName    string
	ProcessDetail pagedata.ProcessDetail
}

type MergeFile added in v0.5.0

type MergeFile struct {
	OutputPath string // should be basename with .pdf ext
	InputPages []Page // should be absolute paths to pdfs (we'll get pagedata, then work out the image filename)
}

type MergeTask added in v0.5.0

type MergeTask struct {
	MergeFile     MergeFile
	ToDir         string
	ProcessDetail pagedata.ProcessDetail
	SpreadName    string
	Template      string
}

type OverlayCommand

type OverlayCommand struct {
	CoverPath                string //get a cover page from here if it exists, and if cover page exists
	FromPath                 string
	ToPath                   string
	ExamName                 string
	TemplatePath             string
	SpreadName               string
	ProcessDetail            pagedata.ProcessDetail
	Msg                      *chmsg.Messager
	PathDecoration           string //this is the "-ma1" for marker1, "mo2" for moderator 2, "d" for done etc
	OpticalBoxSpread         string
	ReadOpticalBoxes         bool
	AncestorPath             string
	OmitPreviousComments     bool //this is for the checked stage, where we don't want earlier comments
	PropagateTextFieldValues bool // this is for enter active - copy textfield values out of pagedata into enter bar
}

type OverlayTask

type OverlayTask struct {
	InputPath                string
	CoverPath                string //no cover page if empty ""
	AncestorPath             string //no redo of ancestry if empty
	Msg                      *chmsg.Messager
	NewFieldMap              map[int][]pagedata.Field
	OldPageDataMap           map[int]pagedata.PageData //this has the individual bits filled in?
	OutputPath               string
	PageCount                int
	ProcessDetail            pagedata.ProcessDetail
	SpreadName               string
	Template                 string
	Who                      string
	OpticalBoxSpread         string
	ReadOpticalBoxes         bool
	TextFields               map[int]map[string]extract.TextField
	OmitPreviousComments     bool //this is for the checked stage, where we don't want earlier comments
	PropagateTextFieldValues bool // this is for enter active - copy textfield values out of pagedata into enter bar
}

type PDFSummary

type PDFSummary struct {
	CourseCode  string
	PreparedFor string
	ToDo        string
}

type Page added in v0.5.0

type Page struct {
	Path    string
	Message string
}

type PageCollection added in v0.5.0

type PageCollection struct {
	Seen    []PageSummary
	Marked  []PageSummary
	Bad     []PageSummary
	Skipped []PageSummary
}

type PageReport added in v0.5.0

type PageReport struct {
	Error        string `csv:"error"`
	What         string `csv:"what"`
	Who          string `csv:"who"`
	When         string `csv:"when"`
	Original     string `csv:"original"`
	OwnPath      string `csv:"path"`
	WasFor       string `csv:"wasfor"`
	PageNumber   int    `csv:"page"`
	Status       string `csv:"status"`
	MergeMessage string `csv:"message"`
	Comments     string `csv:"comments"`
	IsLinked     bool   `csv:"linked"`
	FirstLink    string `csv:"firstlink"`
	LastLink     string `csv:"lastlink"`
}

func (*PageReport) String added in v0.5.0

func (p *PageReport) String() string

type PageSummary added in v0.5.0

type PageSummary struct {
	Original   string //unique key (e.g. original path)
	PageNumber int
	OwnPath    string
	Status     string // pageSeen .. pogeSkipped
	WasFor     string // e.g. marker initials
}

type Q added in v0.6.0

type Q struct {
	Number  string
	Section string
	Mark    string
}

type QuestionSub added in v0.8.4

type QuestionSub struct {
	OldQ string `csv:"oldQ"`
	NewQ string `csv:"newQ"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL