diff --git a/go.mod b/go.mod index f53b2c2..70f9a54 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/UTDNebula/nebula-api/api v0.0.0-20260525053158-3209b0868dcf github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc github.com/chromedp/chromedp v0.15.1 + github.com/cphaensch/go-packrat/v2 v2.1.1 github.com/dongri/phonenumber v0.1.12 github.com/getsentry/sentry-go v0.46.2 github.com/google/go-cmp v0.7.0 diff --git a/go.sum b/go.sum index 3a46081..b84f499 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipw github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cphaensch/go-packrat/v2 v2.1.1 h1:93C9KEwQJ4yGNx0VDIyXG7kqf534+77cihCqIBXQdDs= +github.com/cphaensch/go-packrat/v2 v2.1.1/go.mod h1:kFr8iPj4Am06hrzkVm4KK21H6uDQG5KAT5SI0mLpQQo= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dongri/phonenumber v0.1.12 h1:rR/4VZzxqpocUdyM4dIdfY0TWd8FcW43oiyPaOUxNIk= diff --git a/parser/parser.go b/parser/parser.go index 1dd2241..568e1f9 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -82,9 +82,6 @@ func Parse(inDir string, outDir string, csvPath string, skipValidation bool) { log.Print("\nParsing course requisites...") - // Initialize matchers at runtime for requisite parsing; this is necessary to avoid circular reference errors with compile-time initialization - initMatchers() - for _, course := range Courses { ReqParsers[course.Id]() } diff --git a/parser/requisiteParser.go b/parser/requisiteParser.go index 88f6ff9..95d7d1a 100644 --- a/parser/requisiteParser.go +++ b/parser/requisiteParser.go @@ -1,3 +1,5 @@ +// Package parser provides functionality to parse academic requirement texts (prerequisites, corequisites, etc.) +// into structured requirement objects using a packrat parser and regex-based matchers. package parser import ( @@ -8,51 +10,347 @@ import ( "strings" "github.com/PuerkitoBio/goquery" - "github.com/UTDNebula/api-tools/utils" "github.com/UTDNebula/nebula-api/api/schema" + packrat "github.com/cphaensch/go-packrat/v2" ) -/* - Below is the code for the requisite parser. It is *by far* the most complicated code in this entire project. - In summary, it uses a bottom-up "stack"-based parsing technique, building requisites by taking small groups of text, parsing those groups, - storing them on the "stack", and then uses those previously parsed groups as dependencies for parsing the larger "higher level" groups. +// Requisite is an empty interface that represents any parsed requirement type. +// Concrete types include *schema.CourseRequirement, *schema.CollectionRequirement, etc. +type Requisite interface{} + +// Definition describes a single parser rule: its name, regex patterns, how many capture groups to expect, +// a matcher function that produces a Requisite, and flags for whitespace/case handling. +type Definition struct { + Name string + Patterns []string + GroupCount int + Matcher func(group string, subgroups []string) interface{} + SkipWhitespace bool + CaseSensitive bool +} - It's worth noting that I say stack in quotes above because it's not treated as strictly LIFO like a stack would normally be. -*/ +// registry holds all registered Definition instances. They are converted to packrat parsers in init(). +var registry []*Definition + +// ExprParser is the top‑level packrat parser that combines all registered leaf parsers. +var ExprParser packrat.Parser[Requisite] + +// ParseRequirement is the main entry point for parsing requirement text. +// It handles AND/OR at the top level by splitting the string, then uses +// the packrat parser for the individual parts. +func ParseRequirement(text string) *schema.CollectionRequirement { + fmt.Printf("Parsing input: %q\n", text) // Add this at function start + text = strings.TrimSpace(text) + + flatText, groups := groupParens(text) + groupReqs := make([]interface{}, len(groups)) + // Parse groups bottom‑up (they may contain nested @N references) + for i := len(groups) - 1; i >= 0; i-- { + req := ParseRequirement(groups[i]) // recursive call + if req != nil { + groupReqs[i] = req + } + } + text = flatText + + // Handle AND chains + if strings.Contains(text, " and ") { + parts := strings.Split(text, " and ") + options := make([]interface{}, 0, len(parts)) + for _, part := range parts { + var req interface{} + if placeholder := getPlaceholder(part, groupReqs); placeholder != nil { + req = placeholder + } else { + req = parseLeaf(part) + } + if req != nil && !reqIsThrowaway(req) { + options = append(options, req) + } + } + // AND means all options are required + return schema.NewCollectionRequirement("", len(options), options) + } -// Matcher defines a regex-driven handler used during requisite group parsing. -type Matcher struct { - Regex *regexp.Regexp - Handler func(string, []string) interface{} + // Handle OR chains + if strings.Contains(text, " or ") { + parts := strings.Split(text, " or ") + options := make([]interface{}, 0, len(parts)) + for _, part := range parts { + var req interface{} + if placeholder := getPlaceholder(part, groupReqs); placeholder != nil { + req = placeholder + } else { + req = parseLeaf(part) + } + if req != nil && !reqIsThrowaway(req) { + options = append(options, req) + } + } + // OR means at least one option is required + return schema.NewCollectionRequirement("", 1, options) + } + + // No AND/OR – parse as a single leaf requirement + leaf := parseLeaf(text) + if leaf == nil || reqIsThrowaway(leaf) { + return nil + } + // Wrap as a collection with one option, required = 1 + return schema.NewCollectionRequirement("", 1, []interface{}{leaf}) } -////////////////////// BEGIN MATCHER FUNCS ////////////////////// +// getPlaceholder checks if a string is a group reference like "@3" and returns the corresponding parsed requirement. +func getPlaceholder(part string, groupReqs []interface{}) interface{} { + if strings.HasPrefix(part, "@") && len(part) > 1 { + idx, err := strconv.Atoi(part[1:]) + if err == nil && idx >= 0 && idx < len(groupReqs) { + return groupReqs[idx] + } + } + return nil +} -var ANDRegex = regexp.MustCompile(`(?i)\s+and\s+`) +// parseLeaf uses the packrat parser (which now contains only leaf parsers) to parse a single atomic requirement. +func parseLeaf(input string) interface{} { + scanner := packrat.NewScanner[Requisite](input, packrat.SkipWhitespaceRegex) + node, ok := ExprParser.Match(scanner) + if ok && node.Payload != nil { + return node.Payload + } + return nil +} -// ANDMatcher parses conjunction-separated requisites into an AND collection requirement. -func ANDMatcher(group string, subgroups []string) interface{} { - // Split text along " and " boundaries, then parse subexpressions as groups into an "AND" CollectionRequirement - subExpressions := ANDRegex.Split(group, -1) - parsedSubExps := make([]interface{}, 0, len(subExpressions)) - for _, exp := range subExpressions { - parsedExp := parseGroup(utils.TrimWhitespace(exp)) - // Don't include throwaways - if !reqIsThrowaway(parsedExp) { - parsedSubExps = append(parsedSubExps, parsedExp) +func Register(d *Definition) { + // Check if regex pattern is valid before appending + for _, pat := range d.Patterns { + _, err := regexp.Compile(pat) + if err != nil { + log.Printf("Invalid regex pattern '%s' %v", pat, err) } } + registry = append(registry, d) +} + +// ToParser converts a Definition into a packrat.Parser that tries all its patterns in order. +func (d *Definition) ToParser() packrat.Parser[Requisite] { + // Build an OrParser over each pattern + var subParsers []packrat.Parser[Requisite] + for _, pat := range d.Patterns { + p := packrat.NewRegexParser( + func(s string) Requisite { + utils.VPrintf("[ToParser] Converting %s to parser", d.Name) + re := regexp.MustCompile(pat) + sub := re.FindStringSubmatch(s) + if len(sub) >= d.GroupCount { + utils.VPrintf("[ToParser] Match success: input=%q groups=%v", s, sub) + return d.Matcher(s, sub) + } + utils.VPrintf("[ToParser] Match fail: input=%q (groups=%d, need %d)", s, len(sub), d.GroupCount) + return nil + }, + pat, d.SkipWhitespace, d.CaseSensitive, + ) + subParsers = append(subParsers, p) + } - parsedSubExps = joinAdjacentOthers(parsedSubExps, " and ") + return packrat.NewOrParser(subParsers...) +} - if len(parsedSubExps) > 1 { - return schema.NewCollectionRequirement("AND", len(parsedSubExps), parsedSubExps) - } else { - return parsedSubExps[0] +// init builds the global ExprParser by registering all known requirement patterns. +func init() { + Register(&Definition{ + Name: "ThrowawayParser", + Patterns: []string{`^(?i)(?:better|\d-\d|same as.+)$`}, + GroupCount: 1, + Matcher: ThrowawayMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "OtherParser", + Patterns: []string{ + fmt.Sprintf(`(?i).+%s\s+only$`, utils.R_YEARS), // * Only + `(?i).+\s+in\s+any\s+combination\s+of\s+.+`, // * in any combination of * + }, + GroupCount: 1, + Matcher: OtherMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "MajorMinorParser", + Patterns: []string{ + // majors and minors only + fmt.Sprintf(`(?i)((%s)\s+majors\s+and\s+minors\s+only)`, utils.R_SUBJECT), + }, + GroupCount: 3, + Matcher: MajorMinorMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "CoreCompletionParser", + Patterns: []string{ + // Completion of [a/an] core [course] + `(?i)(Completion\s+of\s+(?:an?\s+)?(\d{3}).+core(?:\s+course)?)`, + }, + GroupCount: 3, + Matcher: CoreCompletionMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "ChoiceParser", + Patterns: []string{ + // Credit cannot be received for both [courses][,] + `(?i)(Credit\s+cannot\s+be\s+received\s+for\s+both\s+(?:courses)?,?(.+))`, + // Credit cannot be received for both [courses][,] + `(?i)(Credit\s+cannot\s+be\s+received\s+for\s+more\s+than\s+one\s+of.+:(.+))`, + }, + GroupCount: 3, + Matcher: ChoiceMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "SubstitionParser", + Patterns: []string{ + fmt.Sprintf(`^(?i)(%s\s+with\s+a(?:\s+grade)?(?:\s+of)?\s+(%s)\s+or\s+better)`, utils.R_SUBJ_COURSE_CAP, utils.R_GRADE), // [name, number, min grade] + }, + GroupCount: 4, + Matcher: SubstitutionMatcher(CourseMinGradeMatcher), + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "CourseMinGradeParser", + Patterns: []string{ + // with a [minimum] grade of [at least] [a] + fmt.Sprintf(`^(?i)%s\s+with\s+a\s+(?:minimum\s+)?grade\s+of\s+(?:at least\s+)?(?:a\s+)?(%s)$`, utils.R_SUBJ_COURSE_CAP, utils.R_GRADE), // [name, number, min grade] + // A grade of [at least] [a] in + fmt.Sprintf(`^(?i)A\s+grade\s+of(?:\s+at\s+least)?(?:\s+a)?\s+(%s)\s+in\s+%s$`, utils.R_GRADE, utils.R_SUBJ_COURSE_CAP), // [min grade, name, number] + }, + GroupCount: 3, + Matcher: CourseMinGradeMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "CourseParser", + Patterns: []string{fmt.Sprintf(`^\s*%s\s*$`, utils.R_SUBJ_COURSE_CAP)}, + GroupCount: 3, + Matcher: CourseMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "ConsentParser", + Patterns: []string{`^(?i)(.+)\s+consent\s+required`}, + GroupCount: 2, + Matcher: ConsentMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "LimitParser", + Patterns: []string{`^(?i)(\d+)\s+semester\s+credit\s+hours\s+maximum$`}, + GroupCount: 2, + Matcher: LimitMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + // majors only + Register(&Definition{ + Name: "MajorParser", + Patterns: []string{`^(?i)(.+)\s+major(?:s\s+only)?$`}, + GroupCount: 2, + Matcher: MajorMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + // minors only + Register(&Definition{ + Name: "MinorParser", + Patterns: []string{`^(?i)(.+)\s+minor(?:s\s+only)?$`}, + GroupCount: 2, + Matcher: MinorMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "CoreParser", + Patterns: []string{ + // Any semester credit hour course + `^(?i)any\s+(\d+)\s+semester\s+credit\s+hour\s+(\d{3})(?:\s+@\d+)?\s+core(?:\s+course)?$`, + }, + GroupCount: 3, + Matcher: CoreMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "GPAParser", + Patterns: []string{ + // Minimum GPA of + `^(?i)([0-9\.]+) GPA$`, // [GPA] + // Minimum GPA of + `^(?i)(?:minimum\s+)?GPA\s+of\s+([0-9\.]+)$`, + // A university grade point average of at least + `^(?i)a(?:\s+university)?\s+grade\s+point\s+average\s+of(?:\s+at\s+least)?\s+([0-9\.]+)$`, + }, + GroupCount: 2, + Matcher: GPAMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + Register(&Definition{ + Name: "GroupParser", + Patterns: []string{`@(\d+)`}, + GroupCount: 2, + Matcher: GroupTagMatcher, + SkipWhitespace: true, + CaseSensitive: false, + }) + + var parsers []packrat.Parser[Requisite] + for _, p := range registry { + parsers = append(parsers, p.ToParser()) } + + // Combine them (note order matters) + ExprParser = packrat.NewOrParser( + parsers..., + ) } +/* --- Matchers and other functions --- */ + +// Matcher defines a regex-driven handler used during requisite group parsing. +type Matcher struct { + Regex *regexp.Regexp + Handler func(string, []string) interface{} +} + +// Matchers contains the ordered collection of matcher rules applied during requisite parsing. +// NOTE: PARENTHESES ARE OF HIGHEST PRECEDENCE! (This is due to groupParens() handling grouping of parenthesized text before parsing begins) +var Matchers []Matcher + // SubstitutionMatcher returns a matcher that replaces a subgroup with parseFnc's result before parsing the outer group. // For example, "(OPRE 3360 or STAT 3360 or STAT 4351), and JSOM majors and minors only" becomes "... and @N". func SubstitutionMatcher(parseFnc func(string, []string) interface{}) func(string, []string) interface{} { @@ -67,30 +365,6 @@ func SubstitutionMatcher(parseFnc func(string, []string) interface{}) func(strin } } -var ORRegex = regexp.MustCompile(`(?i)\s+or\s+`) - -// ORMatcher parses disjunction-separated requisites into an OR collection requirement. -func ORMatcher(group string, subgroups []string) interface{} { - // Split text along " or " boundaries, then parse subexpressions as groups into an "OR" CollectionRequirement - subExpressions := ORRegex.Split(group, -1) - parsedSubExps := make([]interface{}, 0, len(subExpressions)) - for _, exp := range subExpressions { - parsedExp := parseGroup(utils.TrimWhitespace(exp)) - // Don't include throwaways - if !reqIsThrowaway(parsedExp) { - parsedSubExps = append(parsedSubExps, parsedExp) - } - } - - parsedSubExps = joinAdjacentOthers(parsedSubExps, " or ") - - if len(parsedSubExps) > 1 { - return schema.NewCollectionRequirement("OR", 1, parsedSubExps) - } else { - return parsedSubExps[0] - } -} - // CourseMinGradeMatcher returns a course requirement enforcing a minimum grade when an ICN is found. func CourseMinGradeMatcher(group string, subgroups []string) interface{} { icn, err := findICN(subgroups[1], subgroups[2]) @@ -178,9 +452,6 @@ func ThrowawayMatcher(group string, subgroups []string) interface{} { return schema.Requirement{Type: "throwaway"} } -// Regex for group tags -var groupTagRegex = regexp.MustCompile(`@(\d+)`) - // GroupTagMatcher resolves stack-referenced groups by index. func GroupTagMatcher(group string, subgroups []string) interface{} { groupIndex, err := strconv.Atoi(subgroups[1]) @@ -201,246 +472,70 @@ func OtherMatcher(group string, subgroups []string) interface{} { return schema.NewOtherRequirement(ungroupText(group), "") } -/////////////////////// END MATCHER FUNCS /////////////////////// - -// Matchers contains the ordered collection of matcher rules applied during requisite parsing. -// NOTE: PARENTHESES ARE OF HIGHEST PRECEDENCE! (This is due to groupParens() handling grouping of parenthesized text before parsing begins) -var Matchers []Matcher - -// Must init matchers via function at runtime to avoid compile-time circular definition error -func initMatchers() { - Matchers = []Matcher{ - - // Throwaways - { - regexp.MustCompile(`^(?i)(?:better|\d-\d|same as.+)$`), - ThrowawayMatcher, - }, - - /* TO IMPLEMENT: - - X or Y or ... Z Major/Minor - - SUBJECT NUMBER, SUBJECT NUMBER, ..., or SUBJECT NUMBER - - ... probably many more - - */ - - // * only - { - utils.Regexpf(`(?i).+%s\s+only$`, utils.R_YEARS), - OtherMatcher, - }, - - // * in any combination of * - { - regexp.MustCompile(`(?i).+\s+in\s+any\s+combination\s+of\s+.+`), - OtherMatcher, - }, - - // majors and minors only - { - utils.Regexpf(`(?i)((%s)\s+majors\s+and\s+minors\s+only)`, utils.R_SUBJECT), - SubstitutionMatcher(func(group string, subgroups []string) interface{} { - return MajorMinorMatcher(subgroups[1], subgroups[1:3]) - }), - }, - - // Completion of [a/an] core [course] - { - regexp.MustCompile(`(?i)(Completion\s+of\s+(?:an?\s+)?(\d{3}).+core(?:\s+course)?)`), - SubstitutionMatcher(func(group string, subgroups []string) interface{} { - return CoreCompletionMatcher(subgroups[1], subgroups[1:3]) - }), - }, - - // Credit cannot be received for both [courses][,] - { - regexp.MustCompile(`(?i)(Credit\s+cannot\s+be\s+received\s+for\s+both\s+(?:courses)?,?(.+))`), - SubstitutionMatcher(func(group string, subgroups []string) interface{} { - return ChoiceMatcher(subgroups[1], subgroups[1:3]) - }), - }, - - // Credit cannot be received for more than one of *: - { - regexp.MustCompile(`(?i)(Credit\s+cannot\s+be\s+received\s+for\s+more\s+than\s+one\s+of.+:(.+))`), - SubstitutionMatcher(func(group string, subgroups []string) interface{} { - return ChoiceMatcher(subgroups[1], subgroups[1:3]) - }), - }, - - // Logical & - { - ANDRegex, - ANDMatcher, - }, - - // " with a [grade] [of] or better" - { - utils.Regexpf(`^(?i)(%s\s+with\s+a(?:\s+grade)?(?:\s+of)?\s+(%s)\s+or\s+better)`, utils.R_SUBJ_COURSE_CAP, utils.R_GRADE), // [name, number, min grade] - SubstitutionMatcher(func(group string, subgroups []string) interface{} { - return CourseMinGradeMatcher(subgroups[1], subgroups[1:5]) - }), - }, - - // Logical | - { - ORRegex, - ORMatcher, - }, - - // with a [minimum] grade of [at least] [a] - { - utils.Regexpf(`^(?i)%s\s+with\s+a\s+(?:minimum\s+)?grade\s+of\s+(?:at least\s+)?(?:a\s+)?(%s)$`, utils.R_SUBJ_COURSE_CAP, utils.R_GRADE), // [name, number, min grade] - CourseMinGradeMatcher, - }, - - // A grade of [at least] [a] in - { - utils.Regexpf(`^(?i)A\s+grade\s+of(?:\s+at\s+least)?(?:\s+a)?\s+(%s)\s+in\s+%s$`, utils.R_GRADE, utils.R_SUBJ_COURSE_CAP), // [min grade, name, number] - func(group string, subgroups []string) interface{} { - return CourseMinGradeMatcher(group, []string{subgroups[0], subgroups[2], subgroups[3], subgroups[1]}) - }, - }, - - // - { - utils.Regexpf(`^\s*%s\s*$`, utils.R_SUBJ_COURSE_CAP), // [name, number] - CourseMatcher, - }, - - // consent required - { - regexp.MustCompile(`^(?i)(.+)\s+consent\s+required`), // [granter] - ConsentMatcher, - }, - - // semester credit hours maximum - { - regexp.MustCompile(`^(?i)(\d+)\s+semester\s+credit\s+hours\s+maximum$`), - LimitMatcher, - }, - - // This course may only be repeated for credit hours - { - utils.Regexpf(`^(?:%s\s+)?Repeat\s+Limit\s+-\s+(?:%s|This\s+course)\s+may\s+only\s+be\s+repeated\s+for(?:\s+a\s+maximum\s+of)?\s+(\d+)\s+semester\s+cre?dit\s+hours(?:\s+maximum)?$`, utils.R_SUBJ_COURSE, utils.R_SUBJ_COURSE), - LimitMatcher, - }, - - // majors only - { - regexp.MustCompile(`^(?i)(.+)\s+major(?:s\s+only)?$`), - MajorMatcher, - }, - - // minors only - { - regexp.MustCompile(`^(?i)(.+)\s+minor(?:s\s+only)?$`), - MinorMatcher, - }, - - // Any semester credit hour course - { - regexp.MustCompile(`^(?i)any\s+(\d+)\s+semester\s+credit\s+hour\s+(\d{3})(?:\s+@\d+)?\s+core(?:\s+course)?$`), - CoreMatcher, - }, - - // Minimum GPA of - { - regexp.MustCompile(`^(?i)(?:minimum\s+)?GPA\s+of\s+([0-9\.]+)$`), // [GPA] - GPAMatcher, - }, - - // GPA - { - regexp.MustCompile(`^(?i)([0-9\.]+) GPA$`), // [GPA] - GPAMatcher, - }, - - // A university grade point average of at least - { - regexp.MustCompile(`^(?i)a(?:\s+university)?\s+grade\s+point\s+average\s+of(?:\s+at\s+least)?\s+([0-9\.]+)$`), // [GPA] - GPAMatcher, - }, - - // Group tags (i.e. @1) - { - groupTagRegex, // [group #] - GroupTagMatcher, - }, - } -} - +// preOrCoreqRegexp matches lines like "Prerequisites or Corequisites: ..." or "Corequisites or Prerequisites: ..." var preOrCoreqRegexp *regexp.Regexp = regexp.MustCompile(`(?i)((?:Prerequisites?\s+or\s+corequisites?|Corequisites?\s+or\s+prerequisites?):(.*))`) + +// prereqRegexp matches a standard "Prerequisites: ..." line. var prereqRegexp *regexp.Regexp = regexp.MustCompile(`(?i)(Prerequisites?:(.*))`) -var coreqRegexp *regexp.Regexp = regexp.MustCompile(`(?i)(Corequisites?:(.*))`) -// It is very important that these remain in the same order -- this keeps proper precedence in the below function! -var reqRegexes [3]*regexp.Regexp = [3]*regexp.Regexp{preOrCoreqRegexp, prereqRegexp, coreqRegexp} +// coreqRegexp matches a standard "Corequisites: ..." line. +var coreqRegexp *regexp.Regexp = regexp.MustCompile(`(?i)(Corequisites?:(.*))`) -// Returns a closure that parses the course's requisites +// getReqParser returns a closure that parses the course's requisites from its description or enrollment requirements. func getReqParser(course *schema.Course, hasEnrollmentReqs bool, enrollmentReqs *goquery.Selection) func() { return func() { - // Pointer array to course requisite properties must be in same order as reqRegexes above - courseReqs := [3]**schema.CollectionRequirement{&course.Co_or_pre_requisites, &course.Prerequisites, &course.Corequisites} - // The actual text to check for requisites - var checkText string - // Extract req text from the enrollment req info if it exists, otherwise try using the description + text := course.Description if hasEnrollmentReqs { course.Enrollment_reqs = utils.TrimWhitespace(enrollmentReqs.Text()) - checkText = utils.TrimWhitespace(enrollmentReqs.Text()) - } else { - checkText = course.Description + text = course.Enrollment_reqs } - // Iterate over and parse each type of requisite, populating the course's relevant requisite property - for index, reqPtr := range courseReqs { - reqMatches := reqRegexes[index].FindStringSubmatch(checkText) - if reqMatches != nil { - // Actual useful text is the inner match, index 2 - reqText := reqMatches[2] - // Erase any sub-matches for other requisite types by matching outer text, index 1 - for _, regex := range reqRegexes { - matches := regex.FindStringSubmatch(reqText) - if matches != nil { - reqText = strings.Replace(reqText, matches[1], "", -1) - } - } - // Erase current match from checkText to prevent erroneous duplicated Reqs - checkText = strings.Replace(checkText, reqMatches[1], "", -1) - // Split reqText into chunks based on period-space delimiters - textChunks := strings.Split(utils.TrimWhitespace(reqText), ". ") - parsedChunks := make([]interface{}, 0, len(textChunks)) - // Parse each chunk, then add non-throwaway chunks to parsedChunks - for _, chunk := range textChunks { - // Trim any remaining rightmost periods - chunk = utils.TrimWhitespace(strings.TrimRight(chunk, ".")) - parsedChunk := parseChunk(chunk) - if !reqIsThrowaway(parsedChunk) { - parsedChunks = append(parsedChunks, parsedChunk) - } + + // Process each section in order, removing matched headers as we go + sections := []struct { + regex *regexp.Regexp + dest **schema.CollectionRequirement + }{ + {preOrCoreqRegexp, &course.Co_or_pre_requisites}, + {prereqRegexp, &course.Prerequisites}, + {coreqRegexp, &course.Corequisites}, + } + + for _, sec := range sections { + match := sec.regex.FindStringSubmatch(text) + if match == nil { + continue + } + // match[1] is "Prerequisites: ...", match[2] is the inner text + header, content := match[1], match[2] + text = strings.Replace(text, header, "", 1) // remove this section from further scanning + + // Remove any other requisite headers nested inside the content + for _, other := range sections { + if inner := other.regex.FindStringSubmatch(content); inner != nil { + content = strings.Replace(content, inner[1], "", -1) } - // Build CollectionRequirement from parsed chunks and apply to the course property - if len(parsedChunks) > 0 { - *reqPtr = schema.NewCollectionRequirement("REQUISITES", len(parsedChunks), parsedChunks) + } + + // Split by ". " and parse each sentence + var reqs []interface{} + for _, sentence := range strings.Split(content, ". ") { + sentence = strings.TrimRight(sentence, ".") + if req := ParseRequirement(sentence); req != nil && !reqIsThrowaway(req) { + reqs = append(reqs, req) } } + if len(reqs) > 0 { + *sec.dest = schema.NewCollectionRequirement("REQUISITES", len(reqs), reqs) + } } } } -// Function for pulling all requisite references (reqs referenced via group tags) from text -/* -func getReqRefs(text string) []interface{} { - matches := groupTagRegex.FindAllStringSubmatch(text, -1) - refs := make([]interface{}, len(matches)) - for i, submatches := range matches { - refs[i] = GroupTagMatcher(submatches[0], submatches) - } - return refs -} -*/ +// This is the list of produced requisites. Indices coincide with group indices -- aka group @0 will also be the 0th index of the list since it will be processed first. +var requisiteList []interface{} + +// This is the list of groups that are to be parsed. They are the raw text chunks associated with the reqs above. +var groupList []string // Function for creating a new group by replacing subtext in an existing group, and pushing the new group's info to the req and group list func makeSubgroup(group string, subtext string, requisite interface{}) string { @@ -450,41 +545,6 @@ func makeSubgroup(group string, subtext string, requisite interface{}) string { return newGroup } -// Function for joining adjacent OtherRequirements into one OtherRequirement by joining their descriptions with a string -func joinAdjacentOthers(reqs []interface{}, joinString string) []interface{} { - joinedReqs := make([]interface{}, 0, len(reqs)) - // Temp is a blank OtherRequirement - temp := *schema.NewOtherRequirement("", "") - // Iterate over each existing req - for _, req := range reqs { - // Determine whether req is an OtherRequirement - otherReq, isOtherReq := req.(schema.OtherRequirement) - if !isOtherReq { - // If temp contains data, append its final result to the joinedReqs - if temp.Description != "" { - joinedReqs = append(joinedReqs, temp) - } - // Append the non-OtherRequirement to the joinedReqs - joinedReqs = append(joinedReqs, req) - // Reset temp's description - temp.Description = "" - continue - } - // If temp is blank, and req is an otherReq, use otherReq as the initial value of temp - // Otherwise, join temp's existing description with otherReq's description - if temp.Description == "" { - temp = otherReq - } else { - temp.Description = strings.Join([]string{temp.Description, otherReq.Description}, joinString) - } - } - // If temp contains data, append its final result to the joinedReqs - if temp.Description != "" { - joinedReqs = append(joinedReqs, temp) - } - return joinedReqs -} - // Function for finding the Internal Course Number associated with the course with the specified subject and course number func findICN(subject string, number string) (string, error) { for _, coursePtr := range Courses { @@ -495,12 +555,6 @@ func findICN(subject string, number string) (string, error) { return "ERROR", fmt.Errorf("couldn't find an ICN for %s %s", subject, number) } -// This is the list of produced requisites. Indices coincide with group indices -- aka group @0 will also be the 0th index of the list since it will be processed first. -var requisiteList []interface{} - -// This is the list of groups that are to be parsed. They are the raw text chunks associated with the reqs above. -var groupList []string - // Innermost function for parsing individual text groups (used recursively by some Matchers) func parseGroup(grp string) interface{} { // Make sure we trim any mismatched right parentheses @@ -520,26 +574,6 @@ func parseGroup(grp string) interface{} { return *schema.NewOtherRequirement(ungroupText(grp), "") } -// Outermost function for parsing a chunk of requisite text (potentially containing multiple nested text groups) -func parseChunk(chunk string) interface{} { - utils.VPrintf("\nPARSING CHUNK: '%s'", chunk) - // Extract parenthesized groups from chunk text - parseText, parseGroups := groupParens(chunk) - // Initialize the requisite list and group list - requisiteList = make([]interface{}, 0, len(parseGroups)) - groupList = parseGroups - // Begin recursive group parsing -- order is bottom-up - for _, grp := range parseGroups { - parsedReq := parseGroup(grp) - // Only append requisite to stack if it isn't marked as throwaway - if !reqIsThrowaway(parsedReq) { - requisiteList = append(requisiteList, parsedReq) - } - } - finalGroup := parseGroup(parseText) - return finalGroup -} - // Check whether a requisite is a throwaway or not by trying a type assertion to Requirement func reqIsThrowaway(req interface{}) bool { baseReq, isBaseReq := req.(schema.Requirement) diff --git a/parser/requisiteParser_test.go b/parser/requisiteParser_test.go index f80eb55..662de31 100644 --- a/parser/requisiteParser_test.go +++ b/parser/requisiteParser_test.go @@ -1,490 +1,101 @@ package parser import ( - "reflect" - "strings" + "fmt" "testing" "github.com/UTDNebula/nebula-api/api/schema" - "go.mongodb.org/mongo-driver/bson/primitive" + packrat "github.com/cphaensch/go-packrat/v2" ) -func TestInitMatchers(t *testing.T) { - // Test 1: Initialization - Matchers = nil - initMatchers() - - if Matchers == nil { - t.Error("Matchers should not be nil after initialization") - } - - if Matchers == nil { - t.Error("Matchers contain matchers after initialization") +func TestLeafParsers(t *testing.T) { + // Add mock courses so findICN succeeds + Courses = map[string]*schema.Course{ + "CS 1200": {Subject_prefix: "CS", Course_number: "1200", Internal_course_number: "CS-1200"}, + "MATH 2413": {Subject_prefix: "MATH", Course_number: "2413", Internal_course_number: "MATH-2413"}, } - // Test 2: No regex compilation errors - for i, m := range Matchers { - if m.Regex == nil { - t.Errorf("Matcher %d has nil regex", i) - } - } -} - -func TestGroupParens(t *testing.T) { - tests := []struct { - name string + leafTests := []struct { input string - expected string - groups []string + wantType string }{ - { - name: "No parentheses", - input: "MATH 2417 and PHYS 2125", - expected: "MATH 2417 and PHYS 2125", - groups: []string{}, - }, - { - name: "Single parentheses", - input: "MATH 2417 and (PHYS 2125 or PHYS 2126)", - expected: "MATH 2417 and @0", - groups: []string{"PHYS 2125 or PHYS 2126"}, - }, - { - name: "Nested parentheses", - input: "((A and B) or (C and D))", - expected: "@2", - groups: []string{"A and B", "C and D", "@0 or @1"}, - }, - { - name: "Multiple parentheses", - input: "(A) and (B) or (C)", - expected: "@0 and @1 or @2", - groups: []string{"A", "B", "C"}, - }, - { - name: "Mismatched closing parentheses", - input: "(A and B)) extra text", - expected: "@0) extra text", - groups: []string{"A and B"}, - }, - { - name: "Complex expression", - input: "MATH 2417 and (PHYS 2125 or (PHYS 2126 and CHEM 1311))", - expected: "MATH 2417 and @1", - groups: []string{"PHYS 2126 and CHEM 1311", "PHYS 2125 or @0"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, groups := groupParens(tt.input) - if result != tt.expected { - t.Errorf("groupParens() = %q, want %q", result, tt.expected) + {"CS 1200", "*schema.CourseRequirement"}, + {"Minimum GPA of 3.0", "*schema.GPARequirement"}, + {"Instructor consent required", "*schema.ConsentRequirement"}, + {"CS majors only", "*schema.MajorRequirement"}, + {"CS minors only", "*schema.MinorRequirement"}, + {"Completion of 130 core course", "*schema.CoreRequirement"}, + {"same as CS 1200", "schema.Requirement"}, + } + + for _, tt := range leafTests { + t.Run(tt.input, func(t *testing.T) { + scanner := packrat.NewScanner[Requisite](tt.input, packrat.SkipWhitespaceRegex) + node, ok := ExprParser.Match(scanner) + if !ok { + t.Errorf("Match failed") } - if len(groups) != len(tt.groups) { - t.Errorf("group count = %d, want %d", len(groups), len(tt.groups)) + if node.Payload == nil { + t.Errorf("Payload nil") } - for i, group := range groups { - if i < len(tt.groups) && group != tt.groups[i] { - t.Errorf("group[%d] = %q, want %q", i, group, tt.groups[i]) - } + gotType := getTypeName(node.Payload) + if gotType != tt.wantType { + t.Errorf("Type = %s, want %s", gotType, tt.wantType) } }) } } -func TestUngroupText(t *testing.T) { - // Setup groupList as it would be after parsing - groupList = []string{ - "PHYS 2125 or PHYS 2126", - "A and B", - "C and D", - } - - tests := []struct { - name string - input string - expected string - }{ - { - name: "No group tags", - input: "MATH 2417 and something", - expected: "MATH 2417 and something", - }, - { - name: "Single group tag", - input: "MATH 2417 and @0", - expected: "MATH 2417 and (PHYS 2125 or PHYS 2126)", - }, - { - name: "Multiple group tags", - input: "@1 or @2", - expected: "(A and B) or (C and D)", - }, - { - name: "Nested replacement", - input: "@0 with extra @1", - expected: "(PHYS 2125 or PHYS 2126) with extra (A and B)", - }, - { - name: "Out of bounds tag", - input: "@9", - expected: "@9", // Should remain unchanged - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := ungroupText(tt.input) - if result != tt.expected { - t.Errorf("ungroupText() = %q, want %q", result, tt.expected) - } - }) - } -} - -func TestJoinAdjacentOthers(t *testing.T) { - testSectionID := primitive.NewObjectID() - - tests := []struct { - name string - input []interface{} - joinString string - expected []interface{} - }{ - { - name: "empty input", - input: []interface{}{}, - joinString: ", ", - expected: []interface{}{}, - }, - { - name: "single other unchanged", - input: []interface{}{ - *schema.NewOtherRequirement("Only one", ""), - }, - joinString: ", ", - expected: []interface{}{ - *schema.NewOtherRequirement("Only one", ""), - }, - }, - { - name: "adjacent others joined", - input: []interface{}{ - *schema.NewOtherRequirement("First", ""), - *schema.NewOtherRequirement("Second", ""), - *schema.NewOtherRequirement("Third", ""), - }, - joinString: " or ", - expected: []interface{}{ - *schema.NewOtherRequirement("First or Second or Third", ""), - }, - }, - { - name: "others at start and end separated by non-other", - input: []interface{}{ - *schema.NewOtherRequirement("Start1", ""), - *schema.NewOtherRequirement("Start2", ""), - schema.NewSectionRequirement(testSectionID), - *schema.NewOtherRequirement("End1", ""), - *schema.NewOtherRequirement("End2", ""), - }, - joinString: " and ", - expected: []interface{}{ - *schema.NewOtherRequirement("Start1 and Start2", ""), - schema.NewSectionRequirement(testSectionID), - *schema.NewOtherRequirement("End1 and End2", ""), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := joinAdjacentOthers(tt.input, tt.joinString) - - if len(result) != len(tt.expected) { - t.Errorf("expected %d items, got %d", len(tt.expected), len(result)) - return // Don't continue if lengths don't match - } - - // Check each item - for i := range result { - switch r := result[i].(type) { - case schema.OtherRequirement: - expected, ok := tt.expected[i].(schema.OtherRequirement) - if !ok { - t.Errorf("position %d: expected OtherRequirement, got %T", i, tt.expected[i]) - continue - } - if r.Description != expected.Description { - t.Errorf("position %d: description mismatch\n got: %q\n want: %q", - i, r.Description, expected.Description) - } - case *schema.SectionRequirement: - expected, ok := tt.expected[i].(*schema.SectionRequirement) - if !ok { - t.Errorf("position %d: expected *SectionRequirement, got %T", i, tt.expected[i]) - continue - } - if r.SectionReference != expected.SectionReference { - t.Errorf("position %d: section reference mismatch", i) - } - default: - if !reflect.DeepEqual(result[i], tt.expected[i]) { - t.Errorf("position %d: item mismatch", i) - } - } - } - }) +func TestCollectionRequirements(t *testing.T) { + // Add mock courses so findICN succeeds + Courses = map[string]*schema.Course{ + "CS 1200": {Subject_prefix: "CS", Course_number: "1200", Internal_course_number: "CS-1200"}, + "MATH 2413": {Subject_prefix: "MATH", Course_number: "2413", Internal_course_number: "MATH-2413"}, + "PHYS 2325": {Subject_prefix: "PHYS", Course_number: "2325", Internal_course_number: "PHYS-2325"}, + "LIT 1301": {Subject_prefix: "LIT", Course_number: "1301", Internal_course_number: "LIT-1301"}, } -} -func TestReqIsThrowaway(t *testing.T) { tests := []struct { - name string - input interface{} - expected bool + input string + required int + optionsLen int + firstOptionType string }{ - { - name: "throwaway requirement returns true", - input: schema.Requirement{Type: "throwaway"}, - expected: true, - }, - { - name: "non-throwaway requirement returns false", - input: schema.Requirement{Type: "course"}, - expected: false, - }, - { - name: "not a requirement returns false", - input: "not a requirement", - expected: false, - }, - { - name: "nil returns false", - input: nil, - expected: false, - }, - { - name: "other requirement type returns false", - input: schema.Requirement{Type: "section"}, - expected: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := reqIsThrowaway(tt.input) - if result != tt.expected { - t.Errorf("reqIsThrowaway(%v) = %v, want %v", tt.input, result, tt.expected) - } - }) - } -} + // Basic + {"CS 1200", 1, 1, "*schema.CourseRequirement"}, + {"CS 1200 and MATH 2413", 2, 2, "*schema.CourseRequirement"}, + {"CS 1200 or MATH 2413", 1, 2, "*schema.CourseRequirement"}, -func TestMakeSubgroup(t *testing.T) { - tests := []struct { - name string - group string - subtext string - requisite interface{} - expectedGroup string - expectedReqLen int - }{ - { - name: "basic text replacement", - group: "Complete MATH 2413", - subtext: "MATH 2413", - requisite: schema.CourseRequirement{ClassReference: "MATH 2413"}, - expectedGroup: "Complete @0", - expectedReqLen: 1, - }, - { - name: "choice requisite replacement", - group: "Credit cannot be received for both courses, @0 and @1", - subtext: "Credit cannot be received for both courses, @0 and @1", - requisite: schema.ChoiceRequirement{ - Requirement: schema.Requirement{Type: "choice"}, - Choices: &schema.CollectionRequirement{ - Requirement: schema.Requirement{Type: "collection"}, - Name: "", - Required: 1, - Options: []interface{}{ - schema.NewCourseRequirement("CSCI0190", ""), - schema.NewCourseRequirement("CSCI0200", ""), - }, - }, - }, - expectedGroup: "@2", - expectedReqLen: 3, - }, - { - name: "core requirement replacement", - group: "Completion of an 010 core course", - subtext: "Completion of an 010 core course", - requisite: schema.CourseRequirement{ClassReference: "010", MinimumGrade: "B"}, - expectedGroup: "@0", - expectedReqLen: 1, - }, - { - name: "multiple occurrence replacement", - group: "CS 2336 and CS 2336 lab", - subtext: "CS 2336", - requisite: schema.CourseRequirement{ClassReference: "CS 2336"}, - expectedGroup: "@0 and @0 lab", - expectedReqLen: 1, - }, - { - name: "subtext not found", - group: "Complete CHEM 1311", - subtext: "PHYS 2325", - requisite: schema.CourseRequirement{ClassReference: "PHYS 2325"}, - expectedGroup: "Complete CHEM 1311", - expectedReqLen: 1, - }, + // Parenthesis + {"CS 1200 and (MATH 2413 or LIT 1301)", 2, 2, "*schema.CourseRequirement"}, + {"(CS 1200 and MATH 2413) or LIT 1301", 1, 2, "*schema.CollectionRequirement"}, + {"(CS 1200 and MATH 2413) or LIT 1301", 1, 2, "*schema.CollectionRequirement"}, + {"(CS 1200 and MATH 2413) or (LIT 1301 and PHYS 2325)", 1, 2, "*schema.CollectionRequirement"}, + {"CS 1200 and ((MATH 2413 or PHYS 2325) and ENGL 1301)", 2, 2, "*schema.CourseRequirement"}, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Reset global variables - requisiteList = []interface{}{} - groupList = []string{} + t.Run(tt.input, func(t *testing.T) { + result := ParseRequirement(tt.input) - // For the choice requisite test, pre-populate with 2 dummy requisites - if tt.name == "choice requisite replacement" { - requisiteList = []interface{}{ - schema.CourseRequirement{ClassReference: "DUMMY1", MinimumGrade: ""}, - schema.CourseRequirement{ClassReference: "DUMMY2", MinimumGrade: ""}, - } + if result == nil { + t.Fatal("nil result") } - - // Call the function - result := makeSubgroup(tt.group, tt.subtext, tt.requisite) - - // Check the returned group - if result != tt.expectedGroup { - t.Errorf("expected group %q, got %q", tt.expectedGroup, result) + if result.Required != tt.required { + t.Errorf("Required = %d, want %d", result.Required, tt.required) } - - // Check the requisite list length - if len(requisiteList) != tt.expectedReqLen { - t.Errorf("expected %d requisites, got %d", tt.expectedReqLen, len(requisiteList)) + if len(result.Options) != tt.optionsLen { + t.Errorf("Options len = %d, want %d", len(result.Options), tt.optionsLen) } - - // Check if requisite was added - if tt.subtext != "" && strings.Contains(tt.group, tt.subtext) && len(requisiteList) > 0 { - if requisiteList[len(requisiteList)-1] != tt.requisite { - t.Errorf("requisite not properly added to list") + if tt.firstOptionType != "" && len(result.Options) > 0 { + if got := getTypeName(result.Options[0]); got != tt.firstOptionType { + t.Errorf("First option type = %s, want %s", got, tt.firstOptionType) } } - - // Check if group was added to groupList - if len(groupList) != 1 { - t.Errorf("expected 1 group in groupList, got %d", len(groupList)) - } else if groupList[0] != result { - t.Errorf("groupList doesn't contain the result") - } }) } } -func TestFindICN(t *testing.T) { - // Save original Courses and ensure cleanup, even if there is error - originalCourses := Courses - t.Cleanup(func() { - Courses = originalCourses - }) - - // Create test course - testCourse := &schema.Course{ - Subject_prefix: "CS", - Course_number: "1337", - Internal_course_number: "ICN001", - } - - tests := []struct { - name string - setup map[string]*schema.Course - subject string - number string - wantICN string - wantErr bool - }{ - { - name: "finds existing course", - setup: map[string]*schema.Course{ - "": testCourse, - }, - subject: "CS", - number: "1337", - wantICN: "ICN001", - wantErr: false, - }, - { - name: "course not found returns error", - setup: map[string]*schema.Course{ - "": testCourse, - }, - subject: "MATH", - number: "101", - wantICN: "ERROR", - wantErr: true, - }, - { - name: "empty courses list returns error", - setup: map[string]*schema.Course{}, - subject: "CS", - number: "1337", - wantICN: "ERROR", - wantErr: true, - }, - { - name: "case sensitive match fails", - setup: map[string]*schema.Course{ - "": testCourse, - }, - subject: "cs", // lowercase - number: "1337", - wantICN: "ERROR", - wantErr: true, - }, - { - name: "multiple courses finds correct one", - setup: map[string]*schema.Course{ - "": testCourse, - ".": { - Subject_prefix: "MATH", - Course_number: "101", - Internal_course_number: "ICN002", - }, - }, - subject: "MATH", - number: "101", - wantICN: "ICN002", - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Set test data - Courses = tt.setup - - gotICN, err := findICN(tt.subject, tt.number) - - // Check error - if tt.wantErr && err == nil { - t.Error("expected error, got nil") - } - if !tt.wantErr && err != nil { - t.Errorf("unexpected error: %v", err) - } - - // Check ICN - if gotICN != tt.wantICN { - t.Errorf("got ICN %q, want %q", gotICN, tt.wantICN) - } - }) - } - +func getTypeName(i interface{}) string { + return fmt.Sprintf("%T", i) }