From 884631a29bd929e231a2f244865cdf34a206e0b5 Mon Sep 17 00:00:00 2001 From: Jean-Marc MEESSEN Date: Fri, 25 Sep 2020 23:13:36 +0200 Subject: [PATCH] Fix time gap issues --- doc/whats_new.md | 7 +- docker/README.md | 13 +- fleprocess/inferTime.go | 34 ++--- fleprocess/inferTime_test.go | 8 +- fleprocess/load_file.go | 56 ++++---- fleprocess/load_file_test.go | 241 ++++++++++++++++++++++++++++++++++- 6 files changed, 308 insertions(+), 51 deletions(-) diff --git a/doc/whats_new.md b/doc/whats_new.md index 388cca9..239918e 100644 --- a/doc/whats_new.md +++ b/doc/whats_new.md @@ -6,9 +6,10 @@ * Date can have several delimiter ("-", "/", ".", or " ") * Partial dates can be entered ("20-9-6" => "2020-09-06") * The new (FLE v3) "DAY" keyword is now supported (increment is 10 max) -* Date, band, and mode can be specified on a same line, even with a QSO -* Correctly process of optional WWFF keyword -* Correct some typos +* Date, band, and mode can be specified on a same line, even within a QSO line +* Correct processing of optional WWFF keyword +* Time is now correctly inferred when start and end of gap is in the same minute +* Correct some typos and bugs ## Previous releases diff --git a/docker/README.md b/docker/README.md index ab8ea15..28edd05 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,9 +11,18 @@ ## Running the container -To start and execute the `` use : `docker run --rm -i --user $(id -u):$(id -g) -v $(pwd):/FLEcli_data on4kjm/flecli `. If no command is specified, help is displayed. +To start and execute the `` use : `docker run --rm -i --user $(id -u):$(id -g) -v "$(pwd)":/FLEcli_data on4kjm/flecli `. If no command is specified, help is displayed. -This little command will create an alias that avoids typing the whole command: `alias FLEcli="docker run --rm --user $(id -u):$(id -g) -v $(pwd):/FLEcli_data on4kjm/flecli"`. To use it, type `FLEcli version` for example. +This bash script (MAC OS or Linux) will do the trick: + +```` +#!/bin/bash + +CURRENT_UID=$(id -u):$(id -g) +docker run --rm -t --user ${CURRENT_UID} -v "$(pwd)":/FLEcli_data on4kjm/flecli:latest "$@" +```` + +By creating an alias like here after, this command can be called from everywhere. `alias FLEcli="~/myDir/docker-FLEcli.sh"`. To use it, type `FLEcli version` for example. Important note: when specifying the path of a file (input or output), it must be relative to the directory the container was started in. diff --git a/fleprocess/inferTime.go b/fleprocess/inferTime.go index 46ac14d..87e5ee1 100644 --- a/fleprocess/inferTime.go +++ b/fleprocess/inferTime.go @@ -54,23 +54,10 @@ func (tb *InferTimeBlock) String() string { //finalizeTimeGap makes the necessary checks and computation func (tb *InferTimeBlock) finalizeTimeGap() error { - //Check that lastRecordedTime and nextValidTime are not null - if tb.lastRecordedTime.IsZero() { - return errors.New("Gap start time is empty") - } - if tb.nextValidTime.IsZero() { - return errors.New("Gap end time is empty") - } - - //Are the two times equal? - if tb.nextValidTime == tb.lastRecordedTime { - return errors.New("The start and end gap times are equal") + if err :=tb.validateTimeGap(); err != nil { + return err } - //Fail if we have a negative time difference - if tb.nextValidTime.Before(tb.lastRecordedTime) { - return errors.New("Gap start time is later than the Gap end time") - } //Compute the gap diff := tb.nextValidTime.Sub(tb.lastRecordedTime) @@ -86,6 +73,23 @@ func (tb *InferTimeBlock) finalizeTimeGap() error { return nil } +//validateTimeGap checks some important assumptions +func (tb *InferTimeBlock) validateTimeGap() error{ + //Check that lastRecordedTime and nextValidTime are not null + if tb.lastRecordedTime.IsZero() { + return errors.New("Gap start time is empty") + } + if tb.nextValidTime.IsZero() { + return errors.New("Gap end time is empty") + } + + //Fail if we have a negative time difference + if tb.nextValidTime.Before(tb.lastRecordedTime) { + return errors.New("Gap start time is later than the Gap end time") + } + return nil +} + //storeTimeGap updates an InferTimeBLock (last valid time, nbr of records without time). It returns true if we reached the end of the time gap. func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, error) { var err error diff --git a/fleprocess/inferTime_test.go b/fleprocess/inferTime_test.go index de4a402..141e5fa 100644 --- a/fleprocess/inferTime_test.go +++ b/fleprocess/inferTime_test.go @@ -168,16 +168,14 @@ func TestInferTimeBlock_computeGaps_noDifference(t *testing.T) { tb := InferTimeBlock{} tb.lastRecordedTime = time.Date(2020, time.May, 24, 14, 00, 0, 0, time.UTC) tb.nextValidTime = time.Date(2020, time.May, 24, 14, 00, 00, 0, time.UTC) + tb.noTimeCount = 2 //When err := tb.finalizeTimeGap() //Then - if err == nil { - t.Error("Should have failed with an error") - } - if err.Error() != "The start and end gap times are equal" { - t.Errorf("Did not not fail with the expected error. Failed with %s", err) + if err != nil { + t.Errorf("Should not have failed with an error (%s)", err) } } diff --git a/fleprocess/load_file.go b/fleprocess/load_file.go index aac90aa..d5e8efb 100644 --- a/fleprocess/load_file.go +++ b/fleprocess/load_file.go @@ -26,7 +26,7 @@ import ( "time" ) -//LoadFile FIXME +//LoadFile FIXME: //returns nill if failure to process func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogLine, isProcessedOK bool) { file, err := os.Open(inputFilename) @@ -49,6 +49,9 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL file.Close() + //isInferTimeFatalError is set to true is something bad happened while storing time gaps. + isInferTimeFatalError := false + regexpLineComment := regexp.MustCompile("^[[:blank:]]*#") regexpOnlySpaces := regexp.MustCompile("^\\s+$") regexpSingleMultiLineComment := regexp.MustCompile("^[[:blank:]]*{.+}$") @@ -258,7 +261,6 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL previousLogLine.MyGrid = headerMyGrid previousLogLine.QSLmsg = headerQslMsg //previousLogLine.QslMsg is redundant previousLogLine.Nickname = headerNickname - //previousLogLine.Date = headerDate //parse a line logline, errorLine := ParseLine(eachline, previousLogLine) @@ -268,26 +270,22 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL fullLog = append(fullLog, logline) //store time inference data - if isInterpolateTime { + if isInterpolateTime && !isInferTimeFatalError { var isEndOfGap bool if isEndOfGap, err = wrkTimeBlock.storeTimeGap(logline, len(fullLog)); err != nil { - fmt.Println("\nProcessing errors:") - for _, errorLogLine := range errorLog { - fmt.Println(errorLogLine) - } - log.Println("Fatal error: ", err) - os.Exit(1) + errorLog = append(errorLog, fmt.Sprintf("Fatal error at line %d: %s", lineCount, err)) + isInferTimeFatalError = true } //If we reached the end of the time gap, we make the necessary checks and make our gap calculation if isEndOfGap { if err := wrkTimeBlock.finalizeTimeGap(); err != nil { //If an error occured it is a fatal error - fmt.Println("\nProcessing errors:") - for _, errorLogLine := range errorLog { - fmt.Println(errorLogLine) - } - log.Println("Fatal error: ", err) - os.Exit(1) + errorLog = append(errorLog, fmt.Sprintf("Fatal error at line %d: %s", lineCount, err)) + isInferTimeFatalError = true + } + + if isInferTimeFatalError { + break } //add it to the gap collection @@ -320,16 +318,24 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL //if asked to infer the date, lets update the loaded logfile accordingly if isInterpolateTime { - for _, timeBlock := range missingTimeBlockList { - for i := 0; i < timeBlock.noTimeCount; i++ { - position := timeBlock.logFilePosition + i - pLogLine := &fullLog[position] - - // durationOffset := time.Second * time.Duration(timeBlock.deltatime*(i+1)) - durationOffset := timeBlock.deltatime * time.Duration(i+1) - newTime := timeBlock.lastRecordedTime.Add(durationOffset) - updatedTimeString := newTime.Format("1504") - pLogLine.Time = updatedTimeString + //Do we have an open timeBlok that has not been closed. + if (wrkTimeBlock.noTimeCount > 0) && (wrkTimeBlock.nextValidTime.IsZero()) { + errorLog = append(errorLog, fmt.Sprint("Fatal error: missing new time to infer time")) + } else { + for _, timeBlock := range missingTimeBlockList { + if err := timeBlock.validateTimeGap(); err != nil { + errorLog = append(errorLog, fmt.Sprintf("Fatal error: %s", err)) + break + } + for i := 0; i < timeBlock.noTimeCount; i++ { + position := timeBlock.logFilePosition + i + pLogLine := &fullLog[position] + + durationOffset := timeBlock.deltatime * time.Duration(i+1) + newTime := timeBlock.lastRecordedTime.Add(durationOffset) + updatedTimeString := newTime.Format("1504") + pLogLine.Time = updatedTimeString + } } } } diff --git a/fleprocess/load_file_test.go b/fleprocess/load_file_test.go index 4d2b48a..ee7df33 100644 --- a/fleprocess/load_file_test.go +++ b/fleprocess/load_file_test.go @@ -782,7 +782,246 @@ func TestLoadFile_wrongHeader(t *testing.T) { os.Remove(temporaryDataFileName) } -//TODO: if the first call is wrong the infertime doesn't work +//if the first call is wrong the infertime doesn't work +func TestLoadFile_InferTime_missingStartTime(t *testing.T) { + + //Given + dataArray := make([]string, 0) + dataArray = append(dataArray, "# Header") + dataArray = append(dataArray, "myCall on4kjm/p") + dataArray = append(dataArray, "operator on4kjm") + dataArray = append(dataArray, "myWwff onff-0001") + dataArray = append(dataArray, " ") + dataArray = append(dataArray, " #Log") + dataArray = append(dataArray, "date 2020-05-23") + dataArray = append(dataArray, "40m cw ik5zve 9 5") + dataArray = append(dataArray, "on6zq") + dataArray = append(dataArray, "40m 0954 on4do") + + temporaryDataFileName := createTestFile(dataArray) + + //When + loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true) + + //Then + if isLoadedOK { + t.Error("Test file processing should return with an error") + } + if len(loadedLogFile) == 0 { + t.Error("No data loaded") + } + + expectedValue := "ON4KJM/P" + if loadedLogFile[0].MyCall != expectedValue { + t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue) + } + expectedValue = "ON4KJM" + if loadedLogFile[0].Operator != expectedValue { + t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue) + } + expectedValue = "ONFF-0001" + if loadedLogFile[0].MyWWFF != expectedValue { + t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue) + } + + expectedValue = "IK5ZVE" + if loadedLogFile[0].Call != expectedValue { + t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue) + } + expectedValue = "" + if loadedLogFile[0].Time != expectedValue { + t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue) + } + expectedValue = "2020-05-23" + if loadedLogFile[0].Date != expectedValue { + t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue) + } + expectedValue = "ON6ZQ" + if loadedLogFile[1].Call != expectedValue { + t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue) + } + expectedValue = "" + if loadedLogFile[1].Time != expectedValue { + t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue) + } + expectedValue = "ON4DO" + if loadedLogFile[2].Call != expectedValue { + t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue) + } + expectedValue = "0954" + if loadedLogFile[2].Time != expectedValue { + t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue) + } + //Clean Up + os.Remove(temporaryDataFileName) +} + +func TestLoadFile_InferTime_missingEndTime(t *testing.T) { + + //Given + dataArray := make([]string, 0) + dataArray = append(dataArray, "# Header") + dataArray = append(dataArray, "myCall on4kjm/p") + dataArray = append(dataArray, "operator on4kjm") + dataArray = append(dataArray, "myWwff onff-0001") + dataArray = append(dataArray, " ") + dataArray = append(dataArray, " #Log") + dataArray = append(dataArray, "date 2020-05-23") + dataArray = append(dataArray, "40m cw 0950 ik5zve 9 5") + dataArray = append(dataArray, "on6zq") + dataArray = append(dataArray, "40m on4do") + + temporaryDataFileName := createTestFile(dataArray) + + //When + loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true) + + //Then + if isLoadedOK { + t.Error("Test file processing should return with an error") + } + if len(loadedLogFile) == 0 { + t.Error("No data loaded") + } + + expectedValue := "ON4KJM/P" + if loadedLogFile[0].MyCall != expectedValue { + t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue) + } + expectedValue = "ON4KJM" + if loadedLogFile[0].Operator != expectedValue { + t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue) + } + expectedValue = "ONFF-0001" + if loadedLogFile[0].MyWWFF != expectedValue { + t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue) + } + + expectedValue = "IK5ZVE" + if loadedLogFile[0].Call != expectedValue { + t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue) + } + expectedValue = "0950" + if loadedLogFile[0].Time != expectedValue { + t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue) + } + expectedValue = "2020-05-23" + if loadedLogFile[0].Date != expectedValue { + t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue) + } + expectedValue = "ON6ZQ" + if loadedLogFile[1].Call != expectedValue { + t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue) + } + expectedValue = "0950" + if loadedLogFile[1].Time != expectedValue { + t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue) + } + expectedValue = "ON4DO" + if loadedLogFile[2].Call != expectedValue { + t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue) + } + expectedValue = "0950" + if loadedLogFile[2].Time != expectedValue { + t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue) + } + //Clean Up + os.Remove(temporaryDataFileName) +} + +//FIXME: same time +func TestLoadFile_2_QSO_same_time(t *testing.T) { + + //Given + dataArray := make([]string, 0) + dataArray = append(dataArray, "# Header") + dataArray = append(dataArray, "myCall on4kjm/p") + dataArray = append(dataArray, "operator on4kjm") + dataArray = append(dataArray, "myWwff onff-0001") + dataArray = append(dataArray, " ") + dataArray = append(dataArray, " #Log") + dataArray = append(dataArray, "date 2020-05-23") + dataArray = append(dataArray, "40m cw 0950 ik5zve 9 5") + dataArray = append(dataArray, "0951 on6zq") + dataArray = append(dataArray, "f6AA") + dataArray = append(dataArray, "0951 on4do") + dataArray = append(dataArray, "0952 on4bb") + + temporaryDataFileName := createTestFile(dataArray) + + //When + loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true) + + //Then + if !isLoadedOK { + t.Error("Test file should not return with an error") + } + if len(loadedLogFile) == 0 { + t.Error("No data loaded") + } + + expectedValue := "ON4KJM/P" + if loadedLogFile[0].MyCall != expectedValue { + t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue) + } + expectedValue = "ON4KJM" + if loadedLogFile[0].Operator != expectedValue { + t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue) + } + expectedValue = "ONFF-0001" + if loadedLogFile[0].MyWWFF != expectedValue { + t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue) + } + + expectedValue = "IK5ZVE" + if loadedLogFile[0].Call != expectedValue { + t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue) + } + expectedValue = "0950" + if loadedLogFile[0].Time != expectedValue { + t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue) + } + expectedValue = "2020-05-23" + if loadedLogFile[0].Date != expectedValue { + t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue) + } + expectedValue = "ON6ZQ" + if loadedLogFile[1].Call != expectedValue { + t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue) + } + expectedValue = "0951" + if loadedLogFile[1].Time != expectedValue { + t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue) + } + expectedValue = "F6AA" + if loadedLogFile[2].Call != expectedValue { + t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue) + } + expectedValue = "0951" + if loadedLogFile[2].Time != expectedValue { + t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue) + } + expectedValue = "ON4DO" + if loadedLogFile[3].Call != expectedValue { + t.Errorf("Not the expected Call[3] value: %s (expecting %s)", loadedLogFile[3].Call, expectedValue) + } + expectedValue = "0951" + if loadedLogFile[3].Time != expectedValue { + t.Errorf("Not the expected Time[3] value: %s (expecting %s)", loadedLogFile[3].Time, expectedValue) + } + expectedValue = "ON4BB" + if loadedLogFile[4].Call != expectedValue { + t.Errorf("Not the expected Call[4] value: %s (expecting %s)", loadedLogFile[4].Call, expectedValue) + } + expectedValue = "0952" + if loadedLogFile[4].Time != expectedValue { + t.Errorf("Not the expected Time[4] value: %s (expecting %s)", loadedLogFile[4].Time, expectedValue) + } + //Clean Up + os.Remove(temporaryDataFileName) +} + + func TestLoadFile_wrongData(t *testing.T) {