Fix time gap issues

pull/60/head
Jean-Marc MEESSEN 4 years ago committed by GitHub
parent 2e1313ae7b
commit 884631a29b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -6,9 +6,10 @@
* Date can have several delimiter ("-", "/", ".", or " ") * Date can have several delimiter ("-", "/", ".", or " ")
* Partial dates can be entered ("20-9-6" => "2020-09-06") * Partial dates can be entered ("20-9-6" => "2020-09-06")
* The new (FLE v3) "DAY" keyword is now supported (increment is 10 max) * The new (FLE v3) "DAY" keyword is now supported (increment is 10 max)
* Date, band, and mode can be specified on a same line, even with a QSO * Date, band, and mode can be specified on a same line, even within a QSO line
* Correctly process of optional WWFF keyword * Correct processing of optional WWFF keyword
* Correct some typos * Time is now correctly inferred when start and end of gap is in the same minute
* Correct some typos and bugs
## Previous releases ## Previous releases

@ -11,9 +11,18 @@
## Running the container ## Running the container
To start and execute the `<FLEcli command>` use : `docker run --rm -i --user $(id -u):$(id -g) -v $(pwd):/FLEcli_data on4kjm/flecli <FLEcli command>`. If no command is specified, help is displayed. To start and execute the `<FLEcli command>` use : `docker run --rm -i --user $(id -u):$(id -g) -v "$(pwd)":/FLEcli_data on4kjm/flecli <FLEcli command>`. If no command is specified, help is displayed.
This little command will create an alias that avoids typing the whole command: `alias FLEcli="docker run --rm --user $(id -u):$(id -g) -v $(pwd):/FLEcli_data on4kjm/flecli"`. To use it, type `FLEcli version` for example. This bash script (MAC OS or Linux) will do the trick:
````
#!/bin/bash
CURRENT_UID=$(id -u):$(id -g)
docker run --rm -t --user ${CURRENT_UID} -v "$(pwd)":/FLEcli_data on4kjm/flecli:latest "$@"
````
By creating an alias like here after, this command can be called from everywhere. `alias FLEcli="~/myDir/docker-FLEcli.sh"`. To use it, type `FLEcli version` for example.
Important note: when specifying the path of a file (input or output), it must be relative to the directory the container was started in. Important note: when specifying the path of a file (input or output), it must be relative to the directory the container was started in.

@ -54,23 +54,10 @@ func (tb *InferTimeBlock) String() string {
//finalizeTimeGap makes the necessary checks and computation //finalizeTimeGap makes the necessary checks and computation
func (tb *InferTimeBlock) finalizeTimeGap() error { func (tb *InferTimeBlock) finalizeTimeGap() error {
//Check that lastRecordedTime and nextValidTime are not null if err :=tb.validateTimeGap(); err != nil {
if tb.lastRecordedTime.IsZero() { return err
return errors.New("Gap start time is empty")
}
if tb.nextValidTime.IsZero() {
return errors.New("Gap end time is empty")
}
//Are the two times equal?
if tb.nextValidTime == tb.lastRecordedTime {
return errors.New("The start and end gap times are equal")
} }
//Fail if we have a negative time difference
if tb.nextValidTime.Before(tb.lastRecordedTime) {
return errors.New("Gap start time is later than the Gap end time")
}
//Compute the gap //Compute the gap
diff := tb.nextValidTime.Sub(tb.lastRecordedTime) diff := tb.nextValidTime.Sub(tb.lastRecordedTime)
@ -86,6 +73,23 @@ func (tb *InferTimeBlock) finalizeTimeGap() error {
return nil return nil
} }
//validateTimeGap checks some important assumptions
func (tb *InferTimeBlock) validateTimeGap() error{
//Check that lastRecordedTime and nextValidTime are not null
if tb.lastRecordedTime.IsZero() {
return errors.New("Gap start time is empty")
}
if tb.nextValidTime.IsZero() {
return errors.New("Gap end time is empty")
}
//Fail if we have a negative time difference
if tb.nextValidTime.Before(tb.lastRecordedTime) {
return errors.New("Gap start time is later than the Gap end time")
}
return nil
}
//storeTimeGap updates an InferTimeBLock (last valid time, nbr of records without time). It returns true if we reached the end of the time gap. //storeTimeGap updates an InferTimeBLock (last valid time, nbr of records without time). It returns true if we reached the end of the time gap.
func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, error) { func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, error) {
var err error var err error

@ -168,16 +168,14 @@ func TestInferTimeBlock_computeGaps_noDifference(t *testing.T) {
tb := InferTimeBlock{} tb := InferTimeBlock{}
tb.lastRecordedTime = time.Date(2020, time.May, 24, 14, 00, 0, 0, time.UTC) tb.lastRecordedTime = time.Date(2020, time.May, 24, 14, 00, 0, 0, time.UTC)
tb.nextValidTime = time.Date(2020, time.May, 24, 14, 00, 00, 0, time.UTC) tb.nextValidTime = time.Date(2020, time.May, 24, 14, 00, 00, 0, time.UTC)
tb.noTimeCount = 2
//When //When
err := tb.finalizeTimeGap() err := tb.finalizeTimeGap()
//Then //Then
if err == nil { if err != nil {
t.Error("Should have failed with an error") t.Errorf("Should not have failed with an error (%s)", err)
}
if err.Error() != "The start and end gap times are equal" {
t.Errorf("Did not not fail with the expected error. Failed with %s", err)
} }
} }

@ -26,7 +26,7 @@ import (
"time" "time"
) )
//LoadFile FIXME //LoadFile FIXME:
//returns nill if failure to process //returns nill if failure to process
func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogLine, isProcessedOK bool) { func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogLine, isProcessedOK bool) {
file, err := os.Open(inputFilename) file, err := os.Open(inputFilename)
@ -49,6 +49,9 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
file.Close() file.Close()
//isInferTimeFatalError is set to true is something bad happened while storing time gaps.
isInferTimeFatalError := false
regexpLineComment := regexp.MustCompile("^[[:blank:]]*#") regexpLineComment := regexp.MustCompile("^[[:blank:]]*#")
regexpOnlySpaces := regexp.MustCompile("^\\s+$") regexpOnlySpaces := regexp.MustCompile("^\\s+$")
regexpSingleMultiLineComment := regexp.MustCompile("^[[:blank:]]*{.+}$") regexpSingleMultiLineComment := regexp.MustCompile("^[[:blank:]]*{.+}$")
@ -258,7 +261,6 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
previousLogLine.MyGrid = headerMyGrid previousLogLine.MyGrid = headerMyGrid
previousLogLine.QSLmsg = headerQslMsg //previousLogLine.QslMsg is redundant previousLogLine.QSLmsg = headerQslMsg //previousLogLine.QslMsg is redundant
previousLogLine.Nickname = headerNickname previousLogLine.Nickname = headerNickname
//previousLogLine.Date = headerDate
//parse a line //parse a line
logline, errorLine := ParseLine(eachline, previousLogLine) logline, errorLine := ParseLine(eachline, previousLogLine)
@ -268,26 +270,22 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
fullLog = append(fullLog, logline) fullLog = append(fullLog, logline)
//store time inference data //store time inference data
if isInterpolateTime { if isInterpolateTime && !isInferTimeFatalError {
var isEndOfGap bool var isEndOfGap bool
if isEndOfGap, err = wrkTimeBlock.storeTimeGap(logline, len(fullLog)); err != nil { if isEndOfGap, err = wrkTimeBlock.storeTimeGap(logline, len(fullLog)); err != nil {
fmt.Println("\nProcessing errors:") errorLog = append(errorLog, fmt.Sprintf("Fatal error at line %d: %s", lineCount, err))
for _, errorLogLine := range errorLog { isInferTimeFatalError = true
fmt.Println(errorLogLine)
}
log.Println("Fatal error: ", err)
os.Exit(1)
} }
//If we reached the end of the time gap, we make the necessary checks and make our gap calculation //If we reached the end of the time gap, we make the necessary checks and make our gap calculation
if isEndOfGap { if isEndOfGap {
if err := wrkTimeBlock.finalizeTimeGap(); err != nil { if err := wrkTimeBlock.finalizeTimeGap(); err != nil {
//If an error occured it is a fatal error //If an error occured it is a fatal error
fmt.Println("\nProcessing errors:") errorLog = append(errorLog, fmt.Sprintf("Fatal error at line %d: %s", lineCount, err))
for _, errorLogLine := range errorLog { isInferTimeFatalError = true
fmt.Println(errorLogLine) }
}
log.Println("Fatal error: ", err) if isInferTimeFatalError {
os.Exit(1) break
} }
//add it to the gap collection //add it to the gap collection
@ -320,16 +318,24 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
//if asked to infer the date, lets update the loaded logfile accordingly //if asked to infer the date, lets update the loaded logfile accordingly
if isInterpolateTime { if isInterpolateTime {
for _, timeBlock := range missingTimeBlockList { //Do we have an open timeBlok that has not been closed.
for i := 0; i < timeBlock.noTimeCount; i++ { if (wrkTimeBlock.noTimeCount > 0) && (wrkTimeBlock.nextValidTime.IsZero()) {
position := timeBlock.logFilePosition + i errorLog = append(errorLog, fmt.Sprint("Fatal error: missing new time to infer time"))
pLogLine := &fullLog[position] } else {
for _, timeBlock := range missingTimeBlockList {
// durationOffset := time.Second * time.Duration(timeBlock.deltatime*(i+1)) if err := timeBlock.validateTimeGap(); err != nil {
durationOffset := timeBlock.deltatime * time.Duration(i+1) errorLog = append(errorLog, fmt.Sprintf("Fatal error: %s", err))
newTime := timeBlock.lastRecordedTime.Add(durationOffset) break
updatedTimeString := newTime.Format("1504") }
pLogLine.Time = updatedTimeString for i := 0; i < timeBlock.noTimeCount; i++ {
position := timeBlock.logFilePosition + i
pLogLine := &fullLog[position]
durationOffset := timeBlock.deltatime * time.Duration(i+1)
newTime := timeBlock.lastRecordedTime.Add(durationOffset)
updatedTimeString := newTime.Format("1504")
pLogLine.Time = updatedTimeString
}
} }
} }
} }

@ -782,7 +782,246 @@ func TestLoadFile_wrongHeader(t *testing.T) {
os.Remove(temporaryDataFileName) os.Remove(temporaryDataFileName)
} }
//TODO: if the first call is wrong the infertime doesn't work //if the first call is wrong the infertime doesn't work
func TestLoadFile_InferTime_missingStartTime(t *testing.T) {
//Given
dataArray := make([]string, 0)
dataArray = append(dataArray, "# Header")
dataArray = append(dataArray, "myCall on4kjm/p")
dataArray = append(dataArray, "operator on4kjm")
dataArray = append(dataArray, "myWwff onff-0001")
dataArray = append(dataArray, " ")
dataArray = append(dataArray, " #Log")
dataArray = append(dataArray, "date 2020-05-23")
dataArray = append(dataArray, "40m cw ik5zve 9 5")
dataArray = append(dataArray, "on6zq")
dataArray = append(dataArray, "40m 0954 on4do")
temporaryDataFileName := createTestFile(dataArray)
//When
loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true)
//Then
if isLoadedOK {
t.Error("Test file processing should return with an error")
}
if len(loadedLogFile) == 0 {
t.Error("No data loaded")
}
expectedValue := "ON4KJM/P"
if loadedLogFile[0].MyCall != expectedValue {
t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue)
}
expectedValue = "ON4KJM"
if loadedLogFile[0].Operator != expectedValue {
t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue)
}
expectedValue = "ONFF-0001"
if loadedLogFile[0].MyWWFF != expectedValue {
t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
}
expectedValue = "IK5ZVE"
if loadedLogFile[0].Call != expectedValue {
t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
}
expectedValue = ""
if loadedLogFile[0].Time != expectedValue {
t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue)
}
expectedValue = "2020-05-23"
if loadedLogFile[0].Date != expectedValue {
t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue)
}
expectedValue = "ON6ZQ"
if loadedLogFile[1].Call != expectedValue {
t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue)
}
expectedValue = ""
if loadedLogFile[1].Time != expectedValue {
t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue)
}
expectedValue = "ON4DO"
if loadedLogFile[2].Call != expectedValue {
t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue)
}
expectedValue = "0954"
if loadedLogFile[2].Time != expectedValue {
t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue)
}
//Clean Up
os.Remove(temporaryDataFileName)
}
func TestLoadFile_InferTime_missingEndTime(t *testing.T) {
//Given
dataArray := make([]string, 0)
dataArray = append(dataArray, "# Header")
dataArray = append(dataArray, "myCall on4kjm/p")
dataArray = append(dataArray, "operator on4kjm")
dataArray = append(dataArray, "myWwff onff-0001")
dataArray = append(dataArray, " ")
dataArray = append(dataArray, " #Log")
dataArray = append(dataArray, "date 2020-05-23")
dataArray = append(dataArray, "40m cw 0950 ik5zve 9 5")
dataArray = append(dataArray, "on6zq")
dataArray = append(dataArray, "40m on4do")
temporaryDataFileName := createTestFile(dataArray)
//When
loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true)
//Then
if isLoadedOK {
t.Error("Test file processing should return with an error")
}
if len(loadedLogFile) == 0 {
t.Error("No data loaded")
}
expectedValue := "ON4KJM/P"
if loadedLogFile[0].MyCall != expectedValue {
t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue)
}
expectedValue = "ON4KJM"
if loadedLogFile[0].Operator != expectedValue {
t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue)
}
expectedValue = "ONFF-0001"
if loadedLogFile[0].MyWWFF != expectedValue {
t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
}
expectedValue = "IK5ZVE"
if loadedLogFile[0].Call != expectedValue {
t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
}
expectedValue = "0950"
if loadedLogFile[0].Time != expectedValue {
t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue)
}
expectedValue = "2020-05-23"
if loadedLogFile[0].Date != expectedValue {
t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue)
}
expectedValue = "ON6ZQ"
if loadedLogFile[1].Call != expectedValue {
t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue)
}
expectedValue = "0950"
if loadedLogFile[1].Time != expectedValue {
t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue)
}
expectedValue = "ON4DO"
if loadedLogFile[2].Call != expectedValue {
t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue)
}
expectedValue = "0950"
if loadedLogFile[2].Time != expectedValue {
t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue)
}
//Clean Up
os.Remove(temporaryDataFileName)
}
//FIXME: same time
func TestLoadFile_2_QSO_same_time(t *testing.T) {
//Given
dataArray := make([]string, 0)
dataArray = append(dataArray, "# Header")
dataArray = append(dataArray, "myCall on4kjm/p")
dataArray = append(dataArray, "operator on4kjm")
dataArray = append(dataArray, "myWwff onff-0001")
dataArray = append(dataArray, " ")
dataArray = append(dataArray, " #Log")
dataArray = append(dataArray, "date 2020-05-23")
dataArray = append(dataArray, "40m cw 0950 ik5zve 9 5")
dataArray = append(dataArray, "0951 on6zq")
dataArray = append(dataArray, "f6AA")
dataArray = append(dataArray, "0951 on4do")
dataArray = append(dataArray, "0952 on4bb")
temporaryDataFileName := createTestFile(dataArray)
//When
loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true)
//Then
if !isLoadedOK {
t.Error("Test file should not return with an error")
}
if len(loadedLogFile) == 0 {
t.Error("No data loaded")
}
expectedValue := "ON4KJM/P"
if loadedLogFile[0].MyCall != expectedValue {
t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue)
}
expectedValue = "ON4KJM"
if loadedLogFile[0].Operator != expectedValue {
t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue)
}
expectedValue = "ONFF-0001"
if loadedLogFile[0].MyWWFF != expectedValue {
t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
}
expectedValue = "IK5ZVE"
if loadedLogFile[0].Call != expectedValue {
t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
}
expectedValue = "0950"
if loadedLogFile[0].Time != expectedValue {
t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue)
}
expectedValue = "2020-05-23"
if loadedLogFile[0].Date != expectedValue {
t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue)
}
expectedValue = "ON6ZQ"
if loadedLogFile[1].Call != expectedValue {
t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue)
}
expectedValue = "0951"
if loadedLogFile[1].Time != expectedValue {
t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue)
}
expectedValue = "F6AA"
if loadedLogFile[2].Call != expectedValue {
t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue)
}
expectedValue = "0951"
if loadedLogFile[2].Time != expectedValue {
t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue)
}
expectedValue = "ON4DO"
if loadedLogFile[3].Call != expectedValue {
t.Errorf("Not the expected Call[3] value: %s (expecting %s)", loadedLogFile[3].Call, expectedValue)
}
expectedValue = "0951"
if loadedLogFile[3].Time != expectedValue {
t.Errorf("Not the expected Time[3] value: %s (expecting %s)", loadedLogFile[3].Time, expectedValue)
}
expectedValue = "ON4BB"
if loadedLogFile[4].Call != expectedValue {
t.Errorf("Not the expected Call[4] value: %s (expecting %s)", loadedLogFile[4].Call, expectedValue)
}
expectedValue = "0952"
if loadedLogFile[4].Time != expectedValue {
t.Errorf("Not the expected Time[4] value: %s (expecting %s)", loadedLogFile[4].Time, expectedValue)
}
//Clean Up
os.Remove(temporaryDataFileName)
}
func TestLoadFile_wrongData(t *testing.T) { func TestLoadFile_wrongData(t *testing.T) {

Loading…
Cancel
Save