Improve date parsing (multiple delimiter, complete shorthand notation) and DATE marker is optional

fleprocess/inferTime.go

@@ -96,8 +96,12 @@ func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, err
 	if logline.ActualTime != "" {
 		//Are we starting a new block
 		if tb.noTimeCount == 0 {
+			//File is bad: date not found or badly formated
+			if logline.Date == "" {
+				return false, errors.New("Date not defined or badly formated")
+			}
 			if tb.lastRecordedTime, err = time.Parse(ADIFdateTimeFormat, logline.Date+" "+logline.ActualTime); err != nil {
-				log.Println("Fatal error during internal date concersion: ", err)
+				log.Println("Fatal error during internal date conversion: ", err)
 				os.Exit(1)
 			}
 			tb.logFilePosition = position
@@ -107,7 +111,7 @@ func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, err
 				return false, errors.New("Gap start time is empty")
 			}
 			if tb.nextValidTime, err = time.Parse(ADIFdateTimeFormat, logline.Date+" "+logline.ActualTime); err != nil {
-				log.Println("Fatal error during internal date concersion: ", err)
+				log.Println("Fatal error during internal date conversion: ", err)
 				os.Exit(1)
 			}
 			return true, nil

fleprocess/load_file.go

@@ -49,19 +49,20 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
 
 	file.Close()
 
-	regexpLineComment, _ := regexp.Compile("^[[:blank:]]*#")
-	regexpOnlySpaces, _ := regexp.Compile("^\\s+$")
-	regexpSingleMultiLineComment, _ := regexp.Compile("^[[:blank:]]*{.+}$")
-	regexpStartMultiLineComment, _ := regexp.Compile("^[[:blank:]]*{")
-	regexpEndMultiLineComment, _ := regexp.Compile("}$")
-	regexpHeaderMyCall, _ := regexp.Compile("(?i)^mycall ")
-	regexpHeaderOperator, _ := regexp.Compile("(?i)^operator ")
-	regexpHeaderMyWwff, _ := regexp.Compile("(?i)^mywwff ")
-	regexpHeaderMySota, _ := regexp.Compile("(?i)^mysota ")
-	regexpHeaderMyGrid, _ := regexp.Compile("(?i)^mygrid ")
-	regexpHeaderQslMsg, _ := regexp.Compile("(?i)^qslmsg ")
-	regexpHeaderNickname, _ := regexp.Compile("(?i)^nickname ")
-	regexpHeaderDate, _ := regexp.Compile("(?i)^date ")
+	regexpLineComment := regexp.MustCompile("^[[:blank:]]*#")
+	regexpOnlySpaces := regexp.MustCompile("^\\s+$")
+	regexpSingleMultiLineComment := regexp.MustCompile("^[[:blank:]]*{.+}$")
+	regexpStartMultiLineComment := regexp.MustCompile("^[[:blank:]]*{")
+	regexpEndMultiLineComment := regexp.MustCompile("}$")
+	regexpHeaderMyCall := regexp.MustCompile("(?i)^mycall ")
+	regexpHeaderOperator := regexp.MustCompile("(?i)^operator ")
+	regexpHeaderMyWwff := regexp.MustCompile("(?i)^mywwff ")
+	regexpHeaderMySota := regexp.MustCompile("(?i)^mysota ")
+	regexpHeaderMyGrid := regexp.MustCompile("(?i)^mygrid ")
+	regexpHeaderQslMsg := regexp.MustCompile("(?i)^qslmsg ")
+	regexpHeaderNickname := regexp.MustCompile("(?i)^nickname ")
+	regexpHeaderDateMarker := regexp.MustCompile("(?i)^date ")
+	regexpDatePattern := regexp.MustCompile("^(\\d{2}|\\d{4})[-/ .]\\d{1,2}[-/ .]\\d{1,2}$")
 
 	headerMyCall := ""
 	headerOperator := ""
@@ -217,20 +218,43 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
 			continue
 		}
 
-		// Date
-		if regexpHeaderDate.MatchString(eachline) {
+		// Date with keyword
+		if regexpHeaderDateMarker.MatchString(eachline) {
 			errorMsg := ""
-			myDateList := regexpHeaderDate.Split(eachline, -1)
+			myDateList := regexpHeaderDateMarker.Split(eachline, -1)
 			if len(myDateList[1]) > 0 {
-				headerDate, errorMsg = ValidateDate(myDateList[1])
+				normalizedDate := ""
+				normalizedDate, errorMsg = NormalizeDate(myDateList[1])
+				if len(errorMsg) != 0 {
+					errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg))
+				} else {
+					headerDate, errorMsg = ValidateDate(normalizedDate)
 					if len(errorMsg) != 0 {
 						errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, myDateList[1], errorMsg))
 					}
 				}
+			}
 			//If there is no data after the marker, we just skip the data.
 			continue
 		}
 
+		//Date, apparently alone on a line?
+		if regexpDatePattern.MatchString(eachline) {
+			//We probably have a date, let's normalize it
+			errorMsg := ""
+			normalizedDate := ""
+			normalizedDate, errorMsg = NormalizeDate(eachline)
+			if len(errorMsg) != 0 {
+				errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg))
+			} else {
+				headerDate, errorMsg = ValidateDate(normalizedDate)
+				if len(errorMsg) != 0 {
+					errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg))
+				}
+			}
+			continue
+		}
+
 		// ****
 		// ** Process the data block
 		// ****
@@ -256,6 +280,10 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
 			if isInterpolateTime {
 				var isEndOfGap bool
 				if isEndOfGap, err = wrkTimeBlock.storeTimeGap(logline, len(fullLog)); err != nil {
+					fmt.Println("\nProcessing errors:")
+					for _, errorLogLine := range errorLog {
+						fmt.Println(errorLogLine)
+					}
 					log.Println("Fatal error: ", err)
 					os.Exit(1)
 				}
@@ -263,6 +291,10 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
 				if isEndOfGap {
 					if err := wrkTimeBlock.finalizeTimeGap(); err != nil {
 						//If an error occured it is a fatal error
+						fmt.Println("\nProcessing errors:")
+						for _, errorLogLine := range errorLog {
+							fmt.Println(errorLogLine)
+						}
 						log.Println("Fatal error: ", err)
 						os.Exit(1)
 					}

fleprocess/load_file_test.go

@@ -119,6 +119,168 @@ func TestLoadFile_happyCase(t *testing.T) {
 	os.Remove(temporaryDataFileName)
 }
 
+func TestLoadFile_happyCase_date(t *testing.T) {
+
+	//Given
+	dataArray := make([]string, 0)
+	dataArray = append(dataArray, "{ Sample multi-line comment")
+	dataArray = append(dataArray, "	( with quotes) Check: Logging > \"Contest Logging\"")
+	dataArray = append(dataArray, "  - Data item1")
+	dataArray = append(dataArray, "  - Data item2")
+	dataArray = append(dataArray, "  }")
+	dataArray = append(dataArray, "{ one liner comment }")
+	dataArray = append(dataArray, "  { offset one liner comment }")
+	dataArray = append(dataArray, " ")
+	dataArray = append(dataArray, "# Header")
+	dataArray = append(dataArray, "myCall on4kjm/p")
+	dataArray = append(dataArray, "operator on4kjm")
+	dataArray = append(dataArray, "nickname Portable")
+	dataArray = append(dataArray, " ")
+	dataArray = append(dataArray, " #Log")
+	dataArray = append(dataArray, "20/5/23")
+	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
+	dataArray = append(dataArray, "on6zq")
+	dataArray = append(dataArray, "0954 on4do")
+
+	temporaryDataFileName := createTestFile(dataArray)
+
+	//When
+	loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true)
+
+	//Then
+	if !isLoadedOK {
+		t.Error("Test file could not be correctly processed")
+	}
+	if len(loadedLogFile) == 0 {
+		t.Error("No data loaded")
+	}
+
+	expectedValue := "ON4KJM/P"
+	if loadedLogFile[0].MyCall != expectedValue {
+		t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue)
+	}
+	expectedValue = "ON4KJM"
+	if loadedLogFile[0].Operator != expectedValue {
+		t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue)
+	}
+	expectedValue = "Portable"
+	if loadedLogFile[0].Nickname != expectedValue {
+		t.Errorf("Not the expected eQsl Nickname value: %s (expecting %s)", loadedLogFile[0].Nickname, expectedValue)
+	}
+	expectedValue = "IK5ZVE/5"
+	if loadedLogFile[0].Call != expectedValue {
+		t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
+	}
+	expectedValue = "0950"
+	if loadedLogFile[0].Time != expectedValue {
+		t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue)
+	}
+	expectedValue = "2020-05-23"
+	if loadedLogFile[0].Date != expectedValue {
+		t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue)
+	}
+	expectedValue = "ON6ZQ"
+	if loadedLogFile[1].Call != expectedValue {
+		t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue)
+	}
+	expectedValue = "0952"
+	if loadedLogFile[1].Time != expectedValue {
+		t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue)
+	}
+	expectedValue = "ON4DO"
+	if loadedLogFile[2].Call != expectedValue {
+		t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue)
+	}
+	expectedValue = "0954"
+	if loadedLogFile[2].Time != expectedValue {
+		t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue)
+	}
+	//Clean Up
+	os.Remove(temporaryDataFileName)
+}
+
+func TestLoadFile_bad_date(t *testing.T) {
+
+	//Given
+	dataArray := make([]string, 0)
+	dataArray = append(dataArray, "{ Sample multi-line comment")
+	dataArray = append(dataArray, "	( with quotes) Check: Logging > \"Contest Logging\"")
+	dataArray = append(dataArray, "  - Data item1")
+	dataArray = append(dataArray, "  - Data item2")
+	dataArray = append(dataArray, "  }")
+	dataArray = append(dataArray, "{ one liner comment }")
+	dataArray = append(dataArray, "  { offset one liner comment }")
+	dataArray = append(dataArray, " ")
+	dataArray = append(dataArray, "# Header")
+	dataArray = append(dataArray, "myCall on4kjm/p")
+	dataArray = append(dataArray, "operator on4kjm")
+	dataArray = append(dataArray, "nickname Portable")
+	dataArray = append(dataArray, "date 2020-5-18")
+	dataArray = append(dataArray, " ")
+	dataArray = append(dataArray, " #Log")
+	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
+	dataArray = append(dataArray, "20/5/233")
+	dataArray = append(dataArray, "on6zq")
+	dataArray = append(dataArray, "0954 on4do")
+
+	temporaryDataFileName := createTestFile(dataArray)
+
+	//When
+	loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true)
+
+	//Then
+	if isLoadedOK {
+		t.Error("Test file processing should return with an error")
+	}
+	if len(loadedLogFile) == 0 {
+		t.Error("No data loaded")
+	}
+
+	expectedValue := "ON4KJM/P"
+	if loadedLogFile[0].MyCall != expectedValue {
+		t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue)
+	}
+	expectedValue = "ON4KJM"
+	if loadedLogFile[0].Operator != expectedValue {
+		t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue)
+	}
+	expectedValue = "Portable"
+	if loadedLogFile[0].Nickname != expectedValue {
+		t.Errorf("Not the expected eQsl Nickname value: %s (expecting %s)", loadedLogFile[0].Nickname, expectedValue)
+	}
+	expectedValue = "IK5ZVE/5"
+	if loadedLogFile[0].Call != expectedValue {
+		t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
+	}
+	expectedValue = "0950"
+	if loadedLogFile[0].Time != expectedValue {
+		t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue)
+	}
+	expectedValue = "2020-05-18"
+	if loadedLogFile[0].Date != expectedValue {
+		t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue)
+	}
+	expectedValue = "ON6ZQ"
+	if loadedLogFile[1].Call != expectedValue {
+		t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue)
+	}
+	expectedValue = "0952"
+	if loadedLogFile[1].Time != expectedValue {
+		t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue)
+	}
+	expectedValue = "ON4DO"
+	if loadedLogFile[2].Call != expectedValue {
+		t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue)
+	}
+	expectedValue = "0954"
+	if loadedLogFile[2].Time != expectedValue {
+		t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue)
+	}
+	//Clean Up
+	os.Remove(temporaryDataFileName)
+}
+
+
 func TestLoadFile_wrongHeader(t *testing.T) {
 
 	//Given
@@ -142,7 +304,7 @@ func TestLoadFile_wrongHeader(t *testing.T) {
 
 	//Then
 	if isLoadedOK {
-		t.Error("Test file processing should return with aerror")
+		t.Error("Test file processing should return with an error")
 	}
 	if len(loadedLogFile) == 0 {
 		t.Error("No data loaded")

fleprocess/validate.go

@@ -129,6 +129,55 @@ func ValidateCall(sign string) (call, errorMsg string) {
 	return wrongSign, "[" + sign + "] is invalid: too many '/'"
 }
 
+var splitDateRegexp = regexp.MustCompile(`[-/ .]`)
+
+//NormalizeDate takes what looks like a date and normalises it to "YYYY-MM-DD"
+func NormalizeDate(inputStr string) (date, errorMsg string) {
+	//Try to split the string
+	s := splitDateRegexp.Split(inputStr, 4)
+
+	//we should have three and only three elements
+	if i := len(s); i != 3 {
+		errorMsg = fmt.Sprintf("Bad date format: found %d elements while expecting 3.", i)
+		return "*" + inputStr, errorMsg
+	}
+
+
+	//complete the numbers if shorter than expected ("20" for the first and "0" for the two next)
+	year := s[0]
+	if len(year) == 2 {
+		year = "20" + year
+	}
+	//This test is not really necessary, but rather belt and suspenders
+	if len(year) != 4 {
+		errorMsg = "Bad date format: first part doesn't look like a year"
+		return "*" + inputStr, errorMsg
+	}
+
+	month := s[1]
+	if len(month) == 1 {
+		month = "0" + month
+	}
+	if len(month) != 2 {
+		errorMsg = "Bad date format: second part doesn't look like a month"
+		return "*" + inputStr, errorMsg
+	}
+
+	day := s[2]
+	if len(day) == 1 {
+		day = "0" + day
+	}
+	if len(day) != 2 {
+		errorMsg = "Bad date format: third element doesn't look like a day"
+		return "*" + inputStr, errorMsg
+	}
+
+	//re-assemble the string with the correct delimiter
+	date = year + "-" + month + "-" + day
+
+	return date, ""
+}
+
 // ValidateDate verifies whether the string is a valid date (YYYY-MM-DD).
 func ValidateDate(inputStr string) (ref, errorMsg string) {
 

fleprocess/validate_test.go

@@ -381,3 +381,77 @@ func TestValidateGridLocator(t *testing.T) {
 		})
 	}
 }
+
+func TestNormalizeDate(t *testing.T) {
+	type args struct {
+		inputStr string
+	}
+	tests := []struct {
+		name         string
+		args         args
+		wantDate     string
+		wantErrorMsg string
+	}{
+		{
+			"happy case",
+			args{inputStr: "2020-09-04"},
+			"2020-09-04", "",
+		},
+		{
+			"alternate delimiter 1",
+			args{inputStr: "2020/09/04"},
+			"2020-09-04", "",
+		},
+		{
+			"alternate delimiter 2",
+			args{inputStr: "2020.09.04"},
+			"2020-09-04", "",
+		},
+		{
+			"alternate delimiter 3",
+			args{inputStr: "2020 09 04"},
+			"2020-09-04", "",
+		},
+		{
+			"shortened date 1",
+			args{inputStr: "20/09/04"},
+			"2020-09-04", "",
+		},
+		{
+			"shortened date 1",
+			args{inputStr: "2020.9.4"},
+			"2020-09-04", "",
+		},
+		{
+			"Bad date",
+			args{inputStr: "202009.04"},
+			"*202009.04", "Bad date format: found 2 elements while expecting 3.",
+		},
+		{
+			"Bad year length",
+			args{inputStr: "202009.09.15"},
+			"*202009.09.15", "Bad date format: first part doesn't look like a year",
+		},
+		{
+			"Bad month length",
+			args{inputStr: "2020.091.15"},
+			"*2020.091.15", "Bad date format: second part doesn't look like a month",
+		},
+		{
+			"Bad day length",
+			args{inputStr: "2020.09.015"},
+			"*2020.09.015", "Bad date format: third element doesn't look like a day",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotDate, gotErrorMsg := NormalizeDate(tt.args.inputStr)
+			if gotDate != tt.wantDate {
+				t.Errorf("NormalizeDate() gotDate = %v, want %v", gotDate, tt.wantDate)
+			}
+			if gotErrorMsg != tt.wantErrorMsg {
+				t.Errorf("NormalizeDate() gotErrorMsg = %v, want %v", gotErrorMsg, tt.wantErrorMsg)
+			}
+		})
+	}
+}