From ebe782ead0a54292abf6ad191419b858f357b3cf Mon Sep 17 00:00:00 2001 From: Jean-Marc MEESSEN Date: Sat, 5 Sep 2020 22:14:32 +0200 Subject: [PATCH] Improve date parsing (multiple delimiter, complete shorthand notation) and DATE marker is optional --- fleprocess/inferTime.go | 8 +- fleprocess/load_file.go | 68 +++++++++++---- fleprocess/load_file_test.go | 164 ++++++++++++++++++++++++++++++++++- fleprocess/validate.go | 49 +++++++++++ fleprocess/validate_test.go | 74 ++++++++++++++++ 5 files changed, 342 insertions(+), 21 deletions(-) diff --git a/fleprocess/inferTime.go b/fleprocess/inferTime.go index d81533f..46ac14d 100644 --- a/fleprocess/inferTime.go +++ b/fleprocess/inferTime.go @@ -96,8 +96,12 @@ func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, err if logline.ActualTime != "" { //Are we starting a new block if tb.noTimeCount == 0 { + //File is bad: date not found or badly formated + if logline.Date == "" { + return false, errors.New("Date not defined or badly formated") + } if tb.lastRecordedTime, err = time.Parse(ADIFdateTimeFormat, logline.Date+" "+logline.ActualTime); err != nil { - log.Println("Fatal error during internal date concersion: ", err) + log.Println("Fatal error during internal date conversion: ", err) os.Exit(1) } tb.logFilePosition = position @@ -107,7 +111,7 @@ func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, err return false, errors.New("Gap start time is empty") } if tb.nextValidTime, err = time.Parse(ADIFdateTimeFormat, logline.Date+" "+logline.ActualTime); err != nil { - log.Println("Fatal error during internal date concersion: ", err) + log.Println("Fatal error during internal date conversion: ", err) os.Exit(1) } return true, nil diff --git a/fleprocess/load_file.go b/fleprocess/load_file.go index 641d213..aea6254 100644 --- a/fleprocess/load_file.go +++ b/fleprocess/load_file.go @@ -49,19 +49,20 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL file.Close() - regexpLineComment, _ := regexp.Compile("^[[:blank:]]*#") - regexpOnlySpaces, _ := regexp.Compile("^\\s+$") - regexpSingleMultiLineComment, _ := regexp.Compile("^[[:blank:]]*{.+}$") - regexpStartMultiLineComment, _ := regexp.Compile("^[[:blank:]]*{") - regexpEndMultiLineComment, _ := regexp.Compile("}$") - regexpHeaderMyCall, _ := regexp.Compile("(?i)^mycall ") - regexpHeaderOperator, _ := regexp.Compile("(?i)^operator ") - regexpHeaderMyWwff, _ := regexp.Compile("(?i)^mywwff ") - regexpHeaderMySota, _ := regexp.Compile("(?i)^mysota ") - regexpHeaderMyGrid, _ := regexp.Compile("(?i)^mygrid ") - regexpHeaderQslMsg, _ := regexp.Compile("(?i)^qslmsg ") - regexpHeaderNickname, _ := regexp.Compile("(?i)^nickname ") - regexpHeaderDate, _ := regexp.Compile("(?i)^date ") + regexpLineComment := regexp.MustCompile("^[[:blank:]]*#") + regexpOnlySpaces := regexp.MustCompile("^\\s+$") + regexpSingleMultiLineComment := regexp.MustCompile("^[[:blank:]]*{.+}$") + regexpStartMultiLineComment := regexp.MustCompile("^[[:blank:]]*{") + regexpEndMultiLineComment := regexp.MustCompile("}$") + regexpHeaderMyCall := regexp.MustCompile("(?i)^mycall ") + regexpHeaderOperator := regexp.MustCompile("(?i)^operator ") + regexpHeaderMyWwff := regexp.MustCompile("(?i)^mywwff ") + regexpHeaderMySota := regexp.MustCompile("(?i)^mysota ") + regexpHeaderMyGrid := regexp.MustCompile("(?i)^mygrid ") + regexpHeaderQslMsg := regexp.MustCompile("(?i)^qslmsg ") + regexpHeaderNickname := regexp.MustCompile("(?i)^nickname ") + regexpHeaderDateMarker := regexp.MustCompile("(?i)^date ") + regexpDatePattern := regexp.MustCompile("^(\\d{2}|\\d{4})[-/ .]\\d{1,2}[-/ .]\\d{1,2}$") headerMyCall := "" headerOperator := "" @@ -217,20 +218,43 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL continue } - // Date - if regexpHeaderDate.MatchString(eachline) { + // Date with keyword + if regexpHeaderDateMarker.MatchString(eachline) { errorMsg := "" - myDateList := regexpHeaderDate.Split(eachline, -1) + myDateList := regexpHeaderDateMarker.Split(eachline, -1) if len(myDateList[1]) > 0 { - headerDate, errorMsg = ValidateDate(myDateList[1]) + normalizedDate := "" + normalizedDate, errorMsg = NormalizeDate(myDateList[1]) if len(errorMsg) != 0 { - errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, myDateList[1], errorMsg)) + errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg)) + } else { + headerDate, errorMsg = ValidateDate(normalizedDate) + if len(errorMsg) != 0 { + errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, myDateList[1], errorMsg)) + } } } //If there is no data after the marker, we just skip the data. continue } + //Date, apparently alone on a line? + if regexpDatePattern.MatchString(eachline) { + //We probably have a date, let's normalize it + errorMsg := "" + normalizedDate := "" + normalizedDate, errorMsg = NormalizeDate(eachline) + if len(errorMsg) != 0 { + errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg)) + } else { + headerDate, errorMsg = ValidateDate(normalizedDate) + if len(errorMsg) != 0 { + errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg)) + } + } + continue + } + // **** // ** Process the data block // **** @@ -256,6 +280,10 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL if isInterpolateTime { var isEndOfGap bool if isEndOfGap, err = wrkTimeBlock.storeTimeGap(logline, len(fullLog)); err != nil { + fmt.Println("\nProcessing errors:") + for _, errorLogLine := range errorLog { + fmt.Println(errorLogLine) + } log.Println("Fatal error: ", err) os.Exit(1) } @@ -263,6 +291,10 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL if isEndOfGap { if err := wrkTimeBlock.finalizeTimeGap(); err != nil { //If an error occured it is a fatal error + fmt.Println("\nProcessing errors:") + for _, errorLogLine := range errorLog { + fmt.Println(errorLogLine) + } log.Println("Fatal error: ", err) os.Exit(1) } diff --git a/fleprocess/load_file_test.go b/fleprocess/load_file_test.go index 18394b8..525d797 100644 --- a/fleprocess/load_file_test.go +++ b/fleprocess/load_file_test.go @@ -119,6 +119,168 @@ func TestLoadFile_happyCase(t *testing.T) { os.Remove(temporaryDataFileName) } +func TestLoadFile_happyCase_date(t *testing.T) { + + //Given + dataArray := make([]string, 0) + dataArray = append(dataArray, "{ Sample multi-line comment") + dataArray = append(dataArray, " ( with quotes) Check: Logging > \"Contest Logging\"") + dataArray = append(dataArray, " - Data item1") + dataArray = append(dataArray, " - Data item2") + dataArray = append(dataArray, " }") + dataArray = append(dataArray, "{ one liner comment }") + dataArray = append(dataArray, " { offset one liner comment }") + dataArray = append(dataArray, " ") + dataArray = append(dataArray, "# Header") + dataArray = append(dataArray, "myCall on4kjm/p") + dataArray = append(dataArray, "operator on4kjm") + dataArray = append(dataArray, "nickname Portable") + dataArray = append(dataArray, " ") + dataArray = append(dataArray, " #Log") + dataArray = append(dataArray, "20/5/23") + dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5") + dataArray = append(dataArray, "on6zq") + dataArray = append(dataArray, "0954 on4do") + + temporaryDataFileName := createTestFile(dataArray) + + //When + loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true) + + //Then + if !isLoadedOK { + t.Error("Test file could not be correctly processed") + } + if len(loadedLogFile) == 0 { + t.Error("No data loaded") + } + + expectedValue := "ON4KJM/P" + if loadedLogFile[0].MyCall != expectedValue { + t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue) + } + expectedValue = "ON4KJM" + if loadedLogFile[0].Operator != expectedValue { + t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue) + } + expectedValue = "Portable" + if loadedLogFile[0].Nickname != expectedValue { + t.Errorf("Not the expected eQsl Nickname value: %s (expecting %s)", loadedLogFile[0].Nickname, expectedValue) + } + expectedValue = "IK5ZVE/5" + if loadedLogFile[0].Call != expectedValue { + t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue) + } + expectedValue = "0950" + if loadedLogFile[0].Time != expectedValue { + t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue) + } + expectedValue = "2020-05-23" + if loadedLogFile[0].Date != expectedValue { + t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue) + } + expectedValue = "ON6ZQ" + if loadedLogFile[1].Call != expectedValue { + t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue) + } + expectedValue = "0952" + if loadedLogFile[1].Time != expectedValue { + t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue) + } + expectedValue = "ON4DO" + if loadedLogFile[2].Call != expectedValue { + t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue) + } + expectedValue = "0954" + if loadedLogFile[2].Time != expectedValue { + t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue) + } + //Clean Up + os.Remove(temporaryDataFileName) +} + +func TestLoadFile_bad_date(t *testing.T) { + + //Given + dataArray := make([]string, 0) + dataArray = append(dataArray, "{ Sample multi-line comment") + dataArray = append(dataArray, " ( with quotes) Check: Logging > \"Contest Logging\"") + dataArray = append(dataArray, " - Data item1") + dataArray = append(dataArray, " - Data item2") + dataArray = append(dataArray, " }") + dataArray = append(dataArray, "{ one liner comment }") + dataArray = append(dataArray, " { offset one liner comment }") + dataArray = append(dataArray, " ") + dataArray = append(dataArray, "# Header") + dataArray = append(dataArray, "myCall on4kjm/p") + dataArray = append(dataArray, "operator on4kjm") + dataArray = append(dataArray, "nickname Portable") + dataArray = append(dataArray, "date 2020-5-18") + dataArray = append(dataArray, " ") + dataArray = append(dataArray, " #Log") + dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5") + dataArray = append(dataArray, "20/5/233") + dataArray = append(dataArray, "on6zq") + dataArray = append(dataArray, "0954 on4do") + + temporaryDataFileName := createTestFile(dataArray) + + //When + loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true) + + //Then + if isLoadedOK { + t.Error("Test file processing should return with an error") + } + if len(loadedLogFile) == 0 { + t.Error("No data loaded") + } + + expectedValue := "ON4KJM/P" + if loadedLogFile[0].MyCall != expectedValue { + t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue) + } + expectedValue = "ON4KJM" + if loadedLogFile[0].Operator != expectedValue { + t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue) + } + expectedValue = "Portable" + if loadedLogFile[0].Nickname != expectedValue { + t.Errorf("Not the expected eQsl Nickname value: %s (expecting %s)", loadedLogFile[0].Nickname, expectedValue) + } + expectedValue = "IK5ZVE/5" + if loadedLogFile[0].Call != expectedValue { + t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue) + } + expectedValue = "0950" + if loadedLogFile[0].Time != expectedValue { + t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue) + } + expectedValue = "2020-05-18" + if loadedLogFile[0].Date != expectedValue { + t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue) + } + expectedValue = "ON6ZQ" + if loadedLogFile[1].Call != expectedValue { + t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue) + } + expectedValue = "0952" + if loadedLogFile[1].Time != expectedValue { + t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue) + } + expectedValue = "ON4DO" + if loadedLogFile[2].Call != expectedValue { + t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue) + } + expectedValue = "0954" + if loadedLogFile[2].Time != expectedValue { + t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue) + } + //Clean Up + os.Remove(temporaryDataFileName) +} + + func TestLoadFile_wrongHeader(t *testing.T) { //Given @@ -142,7 +304,7 @@ func TestLoadFile_wrongHeader(t *testing.T) { //Then if isLoadedOK { - t.Error("Test file processing should return with aerror") + t.Error("Test file processing should return with an error") } if len(loadedLogFile) == 0 { t.Error("No data loaded") diff --git a/fleprocess/validate.go b/fleprocess/validate.go index 10b0e38..e891fc8 100644 --- a/fleprocess/validate.go +++ b/fleprocess/validate.go @@ -129,6 +129,55 @@ func ValidateCall(sign string) (call, errorMsg string) { return wrongSign, "[" + sign + "] is invalid: too many '/'" } +var splitDateRegexp = regexp.MustCompile(`[-/ .]`) + +//NormalizeDate takes what looks like a date and normalises it to "YYYY-MM-DD" +func NormalizeDate(inputStr string) (date, errorMsg string) { + //Try to split the string + s := splitDateRegexp.Split(inputStr, 4) + + //we should have three and only three elements + if i := len(s); i != 3 { + errorMsg = fmt.Sprintf("Bad date format: found %d elements while expecting 3.", i) + return "*" + inputStr, errorMsg + } + + + //complete the numbers if shorter than expected ("20" for the first and "0" for the two next) + year := s[0] + if len(year) == 2 { + year = "20" + year + } + //This test is not really necessary, but rather belt and suspenders + if len(year) != 4 { + errorMsg = "Bad date format: first part doesn't look like a year" + return "*" + inputStr, errorMsg + } + + month := s[1] + if len(month) == 1 { + month = "0" + month + } + if len(month) != 2 { + errorMsg = "Bad date format: second part doesn't look like a month" + return "*" + inputStr, errorMsg + } + + day := s[2] + if len(day) == 1 { + day = "0" + day + } + if len(day) != 2 { + errorMsg = "Bad date format: third element doesn't look like a day" + return "*" + inputStr, errorMsg + } + + //re-assemble the string with the correct delimiter + date = year + "-" + month + "-" + day + + return date, "" +} + // ValidateDate verifies whether the string is a valid date (YYYY-MM-DD). func ValidateDate(inputStr string) (ref, errorMsg string) { diff --git a/fleprocess/validate_test.go b/fleprocess/validate_test.go index 8601eb5..ccb2052 100644 --- a/fleprocess/validate_test.go +++ b/fleprocess/validate_test.go @@ -381,3 +381,77 @@ func TestValidateGridLocator(t *testing.T) { }) } } + +func TestNormalizeDate(t *testing.T) { + type args struct { + inputStr string + } + tests := []struct { + name string + args args + wantDate string + wantErrorMsg string + }{ + { + "happy case", + args{inputStr: "2020-09-04"}, + "2020-09-04", "", + }, + { + "alternate delimiter 1", + args{inputStr: "2020/09/04"}, + "2020-09-04", "", + }, + { + "alternate delimiter 2", + args{inputStr: "2020.09.04"}, + "2020-09-04", "", + }, + { + "alternate delimiter 3", + args{inputStr: "2020 09 04"}, + "2020-09-04", "", + }, + { + "shortened date 1", + args{inputStr: "20/09/04"}, + "2020-09-04", "", + }, + { + "shortened date 1", + args{inputStr: "2020.9.4"}, + "2020-09-04", "", + }, + { + "Bad date", + args{inputStr: "202009.04"}, + "*202009.04", "Bad date format: found 2 elements while expecting 3.", + }, + { + "Bad year length", + args{inputStr: "202009.09.15"}, + "*202009.09.15", "Bad date format: first part doesn't look like a year", + }, + { + "Bad month length", + args{inputStr: "2020.091.15"}, + "*2020.091.15", "Bad date format: second part doesn't look like a month", + }, + { + "Bad day length", + args{inputStr: "2020.09.015"}, + "*2020.09.015", "Bad date format: third element doesn't look like a day", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotDate, gotErrorMsg := NormalizeDate(tt.args.inputStr) + if gotDate != tt.wantDate { + t.Errorf("NormalizeDate() gotDate = %v, want %v", gotDate, tt.wantDate) + } + if gotErrorMsg != tt.wantErrorMsg { + t.Errorf("NormalizeDate() gotErrorMsg = %v, want %v", gotErrorMsg, tt.wantErrorMsg) + } + }) + } +}