mirror of
https://github.com/on4kjm/FLEcli.git
synced 2025-01-18 13:01:09 +01:00
Improve date parsing (multiple delimiter, complete shorthand notation) and DATE marker is optional
This commit is contained in:
Jean-Marc MEESSEN
GitHub
parent
53bd916d7a
commit
ebe782ead0
5 changed files with 342 additions and 21 deletions
|
|
@ -96,8 +96,12 @@ func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, err
|
|||
if logline.ActualTime != "" {
|
||||
//Are we starting a new block
|
||||
if tb.noTimeCount == 0 {
|
||||
//File is bad: date not found or badly formated
|
||||
if logline.Date == "" {
|
||||
return false, errors.New("Date not defined or badly formated")
|
||||
}
|
||||
if tb.lastRecordedTime, err = time.Parse(ADIFdateTimeFormat, logline.Date+" "+logline.ActualTime); err != nil {
|
||||
log.Println("Fatal error during internal date concersion: ", err)
|
||||
log.Println("Fatal error during internal date conversion: ", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
tb.logFilePosition = position
|
||||
|
|
@ -107,7 +111,7 @@ func (tb *InferTimeBlock) storeTimeGap(logline LogLine, position int) (bool, err
|
|||
return false, errors.New("Gap start time is empty")
|
||||
}
|
||||
if tb.nextValidTime, err = time.Parse(ADIFdateTimeFormat, logline.Date+" "+logline.ActualTime); err != nil {
|
||||
log.Println("Fatal error during internal date concersion: ", err)
|
||||
log.Println("Fatal error during internal date conversion: ", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return true, nil
|
||||
|
|
|
|||
|
|
@ -49,19 +49,20 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
|
|||
|
||||
file.Close()
|
||||
|
||||
regexpLineComment, _ := regexp.Compile("^[[:blank:]]*#")
|
||||
regexpOnlySpaces, _ := regexp.Compile("^\\s+$")
|
||||
regexpSingleMultiLineComment, _ := regexp.Compile("^[[:blank:]]*{.+}$")
|
||||
regexpStartMultiLineComment, _ := regexp.Compile("^[[:blank:]]*{")
|
||||
regexpEndMultiLineComment, _ := regexp.Compile("}$")
|
||||
regexpHeaderMyCall, _ := regexp.Compile("(?i)^mycall ")
|
||||
regexpHeaderOperator, _ := regexp.Compile("(?i)^operator ")
|
||||
regexpHeaderMyWwff, _ := regexp.Compile("(?i)^mywwff ")
|
||||
regexpHeaderMySota, _ := regexp.Compile("(?i)^mysota ")
|
||||
regexpHeaderMyGrid, _ := regexp.Compile("(?i)^mygrid ")
|
||||
regexpHeaderQslMsg, _ := regexp.Compile("(?i)^qslmsg ")
|
||||
regexpHeaderNickname, _ := regexp.Compile("(?i)^nickname ")
|
||||
regexpHeaderDate, _ := regexp.Compile("(?i)^date ")
|
||||
regexpLineComment := regexp.MustCompile("^[[:blank:]]*#")
|
||||
regexpOnlySpaces := regexp.MustCompile("^\\s+$")
|
||||
regexpSingleMultiLineComment := regexp.MustCompile("^[[:blank:]]*{.+}$")
|
||||
regexpStartMultiLineComment := regexp.MustCompile("^[[:blank:]]*{")
|
||||
regexpEndMultiLineComment := regexp.MustCompile("}$")
|
||||
regexpHeaderMyCall := regexp.MustCompile("(?i)^mycall ")
|
||||
regexpHeaderOperator := regexp.MustCompile("(?i)^operator ")
|
||||
regexpHeaderMyWwff := regexp.MustCompile("(?i)^mywwff ")
|
||||
regexpHeaderMySota := regexp.MustCompile("(?i)^mysota ")
|
||||
regexpHeaderMyGrid := regexp.MustCompile("(?i)^mygrid ")
|
||||
regexpHeaderQslMsg := regexp.MustCompile("(?i)^qslmsg ")
|
||||
regexpHeaderNickname := regexp.MustCompile("(?i)^nickname ")
|
||||
regexpHeaderDateMarker := regexp.MustCompile("(?i)^date ")
|
||||
regexpDatePattern := regexp.MustCompile("^(\\d{2}|\\d{4})[-/ .]\\d{1,2}[-/ .]\\d{1,2}$")
|
||||
|
||||
headerMyCall := ""
|
||||
headerOperator := ""
|
||||
|
|
@ -217,20 +218,43 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
|
|||
continue
|
||||
}
|
||||
|
||||
// Date
|
||||
if regexpHeaderDate.MatchString(eachline) {
|
||||
// Date with keyword
|
||||
if regexpHeaderDateMarker.MatchString(eachline) {
|
||||
errorMsg := ""
|
||||
myDateList := regexpHeaderDate.Split(eachline, -1)
|
||||
myDateList := regexpHeaderDateMarker.Split(eachline, -1)
|
||||
if len(myDateList[1]) > 0 {
|
||||
headerDate, errorMsg = ValidateDate(myDateList[1])
|
||||
normalizedDate := ""
|
||||
normalizedDate, errorMsg = NormalizeDate(myDateList[1])
|
||||
if len(errorMsg) != 0 {
|
||||
errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, myDateList[1], errorMsg))
|
||||
errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg))
|
||||
} else {
|
||||
headerDate, errorMsg = ValidateDate(normalizedDate)
|
||||
if len(errorMsg) != 0 {
|
||||
errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, myDateList[1], errorMsg))
|
||||
}
|
||||
}
|
||||
}
|
||||
//If there is no data after the marker, we just skip the data.
|
||||
continue
|
||||
}
|
||||
|
||||
//Date, apparently alone on a line?
|
||||
if regexpDatePattern.MatchString(eachline) {
|
||||
//We probably have a date, let's normalize it
|
||||
errorMsg := ""
|
||||
normalizedDate := ""
|
||||
normalizedDate, errorMsg = NormalizeDate(eachline)
|
||||
if len(errorMsg) != 0 {
|
||||
errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg))
|
||||
} else {
|
||||
headerDate, errorMsg = ValidateDate(normalizedDate)
|
||||
if len(errorMsg) != 0 {
|
||||
errorLog = append(errorLog, fmt.Sprintf("Invalid Date at line %d: %s (%s)", lineCount, eachline, errorMsg))
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// ****
|
||||
// ** Process the data block
|
||||
// ****
|
||||
|
|
@ -256,6 +280,10 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
|
|||
if isInterpolateTime {
|
||||
var isEndOfGap bool
|
||||
if isEndOfGap, err = wrkTimeBlock.storeTimeGap(logline, len(fullLog)); err != nil {
|
||||
fmt.Println("\nProcessing errors:")
|
||||
for _, errorLogLine := range errorLog {
|
||||
fmt.Println(errorLogLine)
|
||||
}
|
||||
log.Println("Fatal error: ", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
|
@ -263,6 +291,10 @@ func LoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []LogL
|
|||
if isEndOfGap {
|
||||
if err := wrkTimeBlock.finalizeTimeGap(); err != nil {
|
||||
//If an error occured it is a fatal error
|
||||
fmt.Println("\nProcessing errors:")
|
||||
for _, errorLogLine := range errorLog {
|
||||
fmt.Println(errorLogLine)
|
||||
}
|
||||
log.Println("Fatal error: ", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -119,6 +119,168 @@ func TestLoadFile_happyCase(t *testing.T) {
|
|||
os.Remove(temporaryDataFileName)
|
||||
}
|
||||
|
||||
func TestLoadFile_happyCase_date(t *testing.T) {
|
||||
|
||||
//Given
|
||||
dataArray := make([]string, 0)
|
||||
dataArray = append(dataArray, "{ Sample multi-line comment")
|
||||
dataArray = append(dataArray, " ( with quotes) Check: Logging > \"Contest Logging\"")
|
||||
dataArray = append(dataArray, " - Data item1")
|
||||
dataArray = append(dataArray, " - Data item2")
|
||||
dataArray = append(dataArray, " }")
|
||||
dataArray = append(dataArray, "{ one liner comment }")
|
||||
dataArray = append(dataArray, " { offset one liner comment }")
|
||||
dataArray = append(dataArray, " ")
|
||||
dataArray = append(dataArray, "# Header")
|
||||
dataArray = append(dataArray, "myCall on4kjm/p")
|
||||
dataArray = append(dataArray, "operator on4kjm")
|
||||
dataArray = append(dataArray, "nickname Portable")
|
||||
dataArray = append(dataArray, " ")
|
||||
dataArray = append(dataArray, " #Log")
|
||||
dataArray = append(dataArray, "20/5/23")
|
||||
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
|
||||
dataArray = append(dataArray, "on6zq")
|
||||
dataArray = append(dataArray, "0954 on4do")
|
||||
|
||||
temporaryDataFileName := createTestFile(dataArray)
|
||||
|
||||
//When
|
||||
loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true)
|
||||
|
||||
//Then
|
||||
if !isLoadedOK {
|
||||
t.Error("Test file could not be correctly processed")
|
||||
}
|
||||
if len(loadedLogFile) == 0 {
|
||||
t.Error("No data loaded")
|
||||
}
|
||||
|
||||
expectedValue := "ON4KJM/P"
|
||||
if loadedLogFile[0].MyCall != expectedValue {
|
||||
t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue)
|
||||
}
|
||||
expectedValue = "ON4KJM"
|
||||
if loadedLogFile[0].Operator != expectedValue {
|
||||
t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue)
|
||||
}
|
||||
expectedValue = "Portable"
|
||||
if loadedLogFile[0].Nickname != expectedValue {
|
||||
t.Errorf("Not the expected eQsl Nickname value: %s (expecting %s)", loadedLogFile[0].Nickname, expectedValue)
|
||||
}
|
||||
expectedValue = "IK5ZVE/5"
|
||||
if loadedLogFile[0].Call != expectedValue {
|
||||
t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
|
||||
}
|
||||
expectedValue = "0950"
|
||||
if loadedLogFile[0].Time != expectedValue {
|
||||
t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue)
|
||||
}
|
||||
expectedValue = "2020-05-23"
|
||||
if loadedLogFile[0].Date != expectedValue {
|
||||
t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue)
|
||||
}
|
||||
expectedValue = "ON6ZQ"
|
||||
if loadedLogFile[1].Call != expectedValue {
|
||||
t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue)
|
||||
}
|
||||
expectedValue = "0952"
|
||||
if loadedLogFile[1].Time != expectedValue {
|
||||
t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue)
|
||||
}
|
||||
expectedValue = "ON4DO"
|
||||
if loadedLogFile[2].Call != expectedValue {
|
||||
t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue)
|
||||
}
|
||||
expectedValue = "0954"
|
||||
if loadedLogFile[2].Time != expectedValue {
|
||||
t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue)
|
||||
}
|
||||
//Clean Up
|
||||
os.Remove(temporaryDataFileName)
|
||||
}
|
||||
|
||||
func TestLoadFile_bad_date(t *testing.T) {
|
||||
|
||||
//Given
|
||||
dataArray := make([]string, 0)
|
||||
dataArray = append(dataArray, "{ Sample multi-line comment")
|
||||
dataArray = append(dataArray, " ( with quotes) Check: Logging > \"Contest Logging\"")
|
||||
dataArray = append(dataArray, " - Data item1")
|
||||
dataArray = append(dataArray, " - Data item2")
|
||||
dataArray = append(dataArray, " }")
|
||||
dataArray = append(dataArray, "{ one liner comment }")
|
||||
dataArray = append(dataArray, " { offset one liner comment }")
|
||||
dataArray = append(dataArray, " ")
|
||||
dataArray = append(dataArray, "# Header")
|
||||
dataArray = append(dataArray, "myCall on4kjm/p")
|
||||
dataArray = append(dataArray, "operator on4kjm")
|
||||
dataArray = append(dataArray, "nickname Portable")
|
||||
dataArray = append(dataArray, "date 2020-5-18")
|
||||
dataArray = append(dataArray, " ")
|
||||
dataArray = append(dataArray, " #Log")
|
||||
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
|
||||
dataArray = append(dataArray, "20/5/233")
|
||||
dataArray = append(dataArray, "on6zq")
|
||||
dataArray = append(dataArray, "0954 on4do")
|
||||
|
||||
temporaryDataFileName := createTestFile(dataArray)
|
||||
|
||||
//When
|
||||
loadedLogFile, isLoadedOK := LoadFile(temporaryDataFileName, true)
|
||||
|
||||
//Then
|
||||
if isLoadedOK {
|
||||
t.Error("Test file processing should return with an error")
|
||||
}
|
||||
if len(loadedLogFile) == 0 {
|
||||
t.Error("No data loaded")
|
||||
}
|
||||
|
||||
expectedValue := "ON4KJM/P"
|
||||
if loadedLogFile[0].MyCall != expectedValue {
|
||||
t.Errorf("Not the expected MyCall value: %s (expecting %s)", loadedLogFile[0].MyCall, expectedValue)
|
||||
}
|
||||
expectedValue = "ON4KJM"
|
||||
if loadedLogFile[0].Operator != expectedValue {
|
||||
t.Errorf("Not the expected Operator value: %s (expecting %s)", loadedLogFile[0].Operator, expectedValue)
|
||||
}
|
||||
expectedValue = "Portable"
|
||||
if loadedLogFile[0].Nickname != expectedValue {
|
||||
t.Errorf("Not the expected eQsl Nickname value: %s (expecting %s)", loadedLogFile[0].Nickname, expectedValue)
|
||||
}
|
||||
expectedValue = "IK5ZVE/5"
|
||||
if loadedLogFile[0].Call != expectedValue {
|
||||
t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
|
||||
}
|
||||
expectedValue = "0950"
|
||||
if loadedLogFile[0].Time != expectedValue {
|
||||
t.Errorf("Not the expected Time[0] value: %s (expecting %s)", loadedLogFile[0].Time, expectedValue)
|
||||
}
|
||||
expectedValue = "2020-05-18"
|
||||
if loadedLogFile[0].Date != expectedValue {
|
||||
t.Errorf("Not the expected Date[0] value: %s (expecting %s)", loadedLogFile[0].Date, expectedValue)
|
||||
}
|
||||
expectedValue = "ON6ZQ"
|
||||
if loadedLogFile[1].Call != expectedValue {
|
||||
t.Errorf("Not the expected Call[1] value: %s (expecting %s)", loadedLogFile[1].Call, expectedValue)
|
||||
}
|
||||
expectedValue = "0952"
|
||||
if loadedLogFile[1].Time != expectedValue {
|
||||
t.Errorf("Not the expected Time[1] value: %s (expecting %s)", loadedLogFile[1].Time, expectedValue)
|
||||
}
|
||||
expectedValue = "ON4DO"
|
||||
if loadedLogFile[2].Call != expectedValue {
|
||||
t.Errorf("Not the expected Call[2] value: %s (expecting %s)", loadedLogFile[2].Call, expectedValue)
|
||||
}
|
||||
expectedValue = "0954"
|
||||
if loadedLogFile[2].Time != expectedValue {
|
||||
t.Errorf("Not the expected Time[2] value: %s (expecting %s)", loadedLogFile[2].Time, expectedValue)
|
||||
}
|
||||
//Clean Up
|
||||
os.Remove(temporaryDataFileName)
|
||||
}
|
||||
|
||||
|
||||
func TestLoadFile_wrongHeader(t *testing.T) {
|
||||
|
||||
//Given
|
||||
|
|
@ -142,7 +304,7 @@ func TestLoadFile_wrongHeader(t *testing.T) {
|
|||
|
||||
//Then
|
||||
if isLoadedOK {
|
||||
t.Error("Test file processing should return with aerror")
|
||||
t.Error("Test file processing should return with an error")
|
||||
}
|
||||
if len(loadedLogFile) == 0 {
|
||||
t.Error("No data loaded")
|
||||
|
|
|
|||
|
|
@ -129,6 +129,55 @@ func ValidateCall(sign string) (call, errorMsg string) {
|
|||
return wrongSign, "[" + sign + "] is invalid: too many '/'"
|
||||
}
|
||||
|
||||
var splitDateRegexp = regexp.MustCompile(`[-/ .]`)
|
||||
|
||||
//NormalizeDate takes what looks like a date and normalises it to "YYYY-MM-DD"
|
||||
func NormalizeDate(inputStr string) (date, errorMsg string) {
|
||||
//Try to split the string
|
||||
s := splitDateRegexp.Split(inputStr, 4)
|
||||
|
||||
//we should have three and only three elements
|
||||
if i := len(s); i != 3 {
|
||||
errorMsg = fmt.Sprintf("Bad date format: found %d elements while expecting 3.", i)
|
||||
return "*" + inputStr, errorMsg
|
||||
}
|
||||
|
||||
|
||||
//complete the numbers if shorter than expected ("20" for the first and "0" for the two next)
|
||||
year := s[0]
|
||||
if len(year) == 2 {
|
||||
year = "20" + year
|
||||
}
|
||||
//This test is not really necessary, but rather belt and suspenders
|
||||
if len(year) != 4 {
|
||||
errorMsg = "Bad date format: first part doesn't look like a year"
|
||||
return "*" + inputStr, errorMsg
|
||||
}
|
||||
|
||||
month := s[1]
|
||||
if len(month) == 1 {
|
||||
month = "0" + month
|
||||
}
|
||||
if len(month) != 2 {
|
||||
errorMsg = "Bad date format: second part doesn't look like a month"
|
||||
return "*" + inputStr, errorMsg
|
||||
}
|
||||
|
||||
day := s[2]
|
||||
if len(day) == 1 {
|
||||
day = "0" + day
|
||||
}
|
||||
if len(day) != 2 {
|
||||
errorMsg = "Bad date format: third element doesn't look like a day"
|
||||
return "*" + inputStr, errorMsg
|
||||
}
|
||||
|
||||
//re-assemble the string with the correct delimiter
|
||||
date = year + "-" + month + "-" + day
|
||||
|
||||
return date, ""
|
||||
}
|
||||
|
||||
// ValidateDate verifies whether the string is a valid date (YYYY-MM-DD).
|
||||
func ValidateDate(inputStr string) (ref, errorMsg string) {
|
||||
|
||||
|
|
|
|||
|
|
@ -381,3 +381,77 @@ func TestValidateGridLocator(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDate(t *testing.T) {
|
||||
type args struct {
|
||||
inputStr string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
wantDate string
|
||||
wantErrorMsg string
|
||||
}{
|
||||
{
|
||||
"happy case",
|
||||
args{inputStr: "2020-09-04"},
|
||||
"2020-09-04", "",
|
||||
},
|
||||
{
|
||||
"alternate delimiter 1",
|
||||
args{inputStr: "2020/09/04"},
|
||||
"2020-09-04", "",
|
||||
},
|
||||
{
|
||||
"alternate delimiter 2",
|
||||
args{inputStr: "2020.09.04"},
|
||||
"2020-09-04", "",
|
||||
},
|
||||
{
|
||||
"alternate delimiter 3",
|
||||
args{inputStr: "2020 09 04"},
|
||||
"2020-09-04", "",
|
||||
},
|
||||
{
|
||||
"shortened date 1",
|
||||
args{inputStr: "20/09/04"},
|
||||
"2020-09-04", "",
|
||||
},
|
||||
{
|
||||
"shortened date 1",
|
||||
args{inputStr: "2020.9.4"},
|
||||
"2020-09-04", "",
|
||||
},
|
||||
{
|
||||
"Bad date",
|
||||
args{inputStr: "202009.04"},
|
||||
"*202009.04", "Bad date format: found 2 elements while expecting 3.",
|
||||
},
|
||||
{
|
||||
"Bad year length",
|
||||
args{inputStr: "202009.09.15"},
|
||||
"*202009.09.15", "Bad date format: first part doesn't look like a year",
|
||||
},
|
||||
{
|
||||
"Bad month length",
|
||||
args{inputStr: "2020.091.15"},
|
||||
"*2020.091.15", "Bad date format: second part doesn't look like a month",
|
||||
},
|
||||
{
|
||||
"Bad day length",
|
||||
args{inputStr: "2020.09.015"},
|
||||
"*2020.09.015", "Bad date format: third element doesn't look like a day",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotDate, gotErrorMsg := NormalizeDate(tt.args.inputStr)
|
||||
if gotDate != tt.wantDate {
|
||||
t.Errorf("NormalizeDate() gotDate = %v, want %v", gotDate, tt.wantDate)
|
||||
}
|
||||
if gotErrorMsg != tt.wantErrorMsg {
|
||||
t.Errorf("NormalizeDate() gotErrorMsg = %v, want %v", gotErrorMsg, tt.wantErrorMsg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue