From ead8f70d8fd1843f2b1606161edbc74dfc1333c3 Mon Sep 17 00:00:00 2001 From: ThatRell Date: Sat, 26 Sep 2026 23:50:01 -0500 Subject: [PATCH 1/7] Replaced panic statements in budgets.go --- scrapers/budgets.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scrapers/budgets.go b/scrapers/budgets.go index 6fde3f9..dd6a0b1 100644 --- a/scrapers/budgets.go +++ b/scrapers/budgets.go @@ -24,7 +24,7 @@ type Budget struct { Href string } -func ScrapeBudgets(outDir string) { +func ScrapeBudgets(outDir string) error { // Start chromedp chromedpCtx, cancel := utils.InitChromeDp() @@ -35,7 +35,7 @@ func ScrapeBudgets(outDir string) { os.RemoveAll(outSubDir) err := os.MkdirAll(outSubDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("Error creating output folder: %w", err) } // Go to listings page @@ -43,7 +43,7 @@ func ScrapeBudgets(outDir string) { chromedp.Navigate(`https://finance.utdallas.edu/for-others/public-reports/`), ) if err != nil { - panic(err) + return fmt.Errorf("Error navigating to listings page: %w", err) } // Selector for the scraping the budget nodes @@ -59,7 +59,7 @@ func ScrapeBudgets(outDir string) { chromedp.Nodes(financialReportSel, &financialReportNodes, chromedp.BySearch), ) if err != nil { - panic(err) + return fmt.Errorf("Error retrieving Financial Report Nodes: %w", err) } links := utils.ExtractTextAndHref(financialReportNodes, chromedpCtx) for _, link := range links { @@ -75,7 +75,7 @@ func ScrapeBudgets(outDir string) { chromedp.Nodes(budgetReportSel, &budgetReportNodes, chromedp.BySearch), ) if err != nil { - panic(err) + return fmt.Errorf("Error retrieving Budget Report Nodes: %w", err) } links = utils.ExtractTextAndHref(budgetReportNodes, chromedpCtx) for _, link := range links { @@ -96,16 +96,18 @@ func ScrapeBudgets(outDir string) { outSubDir, ) } + + return nil } -func downloadPdf(href string, filename string, outDir string) { +func downloadPdf(href string, filename string, outDir string) error { client := &http.Client{ Timeout: 30 * time.Second, } req, err := http.NewRequest("GET", href, nil) if err != nil { - panic(err) + return fmt.Errorf("Error creating request: %w", err) } // Use a user agent and referer to avoid 599 errors @@ -114,13 +116,13 @@ func downloadPdf(href string, filename string, outDir string) { resp, err := client.Do(req) if err != nil { - panic(err) + return fmt.Errorf("Error sending request: %w", err) } defer resp.Body.Close() // Check response if resp.StatusCode != http.StatusOK { - panic(fmt.Errorf("failed to download \"%s\": status code %d", filename, resp.StatusCode)) + return fmt.Errorf("failed to download \"%s\": status code %d", filename, resp.StatusCode) } // Get sub folder from output folder @@ -131,24 +133,25 @@ func downloadPdf(href string, filename string, outDir string) { // Make output folder err = os.MkdirAll(outYearDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("Error creating output folder: %w", err) } // Create blank file out, err := os.Create(filepath.Join(outYearDir, fmt.Sprintf("%s.pdf", filename))) if err != nil { - panic(err) + return fmt.Errorf("Error creating blank file: %w", err) } defer out.Close() // Output response to blank file _, err = io.Copy(out, resp.Body) if err != nil { - log.Printf("Error saving %s: %v", filename, err) - return + return fmt.Errorf("Error saving %s: %w", filename, err) } log.Printf("Scraped budget %s!", filename) time.Sleep(1 * time.Second) + + return nil } From 5c5bdeb51570b6e6074275e22ff3c68d9c099181 Mon Sep 17 00:00:00 2001 From: ThatRell Date: Sun, 27 Sep 2026 00:07:04 -0500 Subject: [PATCH 2/7] Replaced panic statements in cometCalendar.go --- scrapers/cometCalendar.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapers/cometCalendar.go b/scrapers/cometCalendar.go index a461b4e..6080db9 100644 --- a/scrapers/cometCalendar.go +++ b/scrapers/cometCalendar.go @@ -69,10 +69,10 @@ type APICalendarResponse struct { } // ScrapeCometCalendar retrieves calendar events through the API -func ScrapeCometCalendar(outDir string) { +func ScrapeCometCalendar(outDir string) error { err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("Error creating output folder: %w", err) } client := http.Client{ Timeout: 15 * time.Second, @@ -83,7 +83,7 @@ func ScrapeCometCalendar(outDir string) { log.Printf("Getting the number of pages...") if err := callAndUnmarshal(&client, 0, &calendarData); err != nil { - panic(err) + return fmt.Errorf("Error fetching/decoding page %d: %w", 0, err) } numPages := calendarData.Page["total"] log.Printf("The number of pages is %d!\n\n", numPages) @@ -92,13 +92,13 @@ func ScrapeCometCalendar(outDir string) { for page := range numPages { log.Printf("Scraping events of page %d...", page+1) if err := callAndUnmarshal(&client, page+1, &calendarData); err != nil { - panic(err) + return fmt.Errorf("Error fetching/decoding page %d: %w", page+1, err) } for _, event := range calendarData.Events { // Parse all necessary info startTime, endTime, err := getTime(event.Event) if err != nil { - panic(err) + return fmt.Errorf("Error parsing start & end times: %w", err) } if startTime.After(endTime) { fmt.Printf("start: %s, end: %s\n", startTime, endTime) @@ -132,10 +132,12 @@ func ScrapeCometCalendar(outDir string) { writePath := fmt.Sprintf("%s/cometCalendarScraped.json", outDir) if err := utils.WriteJSON(writePath, calendarEvents); err != nil { - panic(err) + return fmt.Errorf("Error writing JSON to path: %w", err) } log.Printf("Finished scraping %d events successfully!\n\n", len(calendarEvents)) + + return nil } // callAndUnmarshal fetches a calendar page and decodes it into data. From 7aefecff20f19d0abe6707b7554f88b22c635ab9 Mon Sep 17 00:00:00 2001 From: ThatRell Date: Sun, 27 Sep 2026 00:42:22 -0500 Subject: [PATCH 3/7] Replaced log.Fatalf() statements in coursebook.go --- scrapers/coursebook.go | 45 ++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/scrapers/coursebook.go b/scrapers/coursebook.go index 539b6a8..299b0ec 100644 --- a/scrapers/coursebook.go +++ b/scrapers/coursebook.go @@ -36,12 +36,13 @@ const ( ) // ScrapeCoursebook scrapes utd coursebook for the provided term (semester) -func ScrapeCoursebook(term string, startPrefix string, outDir string, resume bool) { +func ScrapeCoursebook(term string, startPrefix string, outDir string, resume bool) error { + inputError := errors.New("InvalidInputError") if startPrefix != "" && !prefixRegex.MatchString(startPrefix) { - log.Fatalf("Invalid starting prefix %s, must match format cp_{abcde}", startPrefix) + return fmt.Errorf("Invalid starting prefix %s, must match format cp_{abcde}: %w", startPrefix, inputError) } if !termRegex.MatchString(term) { - log.Fatalf("Invalid term %s, must match format {00-99}{s/f/u}", term) + return fmt.Errorf("Invalid term %s, must match format {00-99}{s/f/u}: %w", term, inputError) } scraper := newCoursebookScraper(term, outDir) @@ -49,7 +50,11 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo if resume && startPrefix == "" { // providing a starting prefix overrides the resume flag - startPrefix = scraper.lastCompletePrefix() + var err error + startPrefix, err = scraper.lastCompletePrefix() + if err != nil { + return fmt.Errorf("Error getting last complete prefix: %w", err) + } } log.Printf("[Begin Scrape] Starting scrape for term %s with %d prefixes", term, len(scraper.prefixes)) @@ -62,7 +67,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo start := time.Now() if err := scraper.ensurePrefixFolder(prefix); err != nil { - log.Fatal(err) + return fmt.Errorf("Error ensuring prefix folder: %w", err) } var sectionIds []string @@ -76,7 +81,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo } if err != nil { - log.Fatalf("Error getting section ids for %s ", prefix) + return fmt.Errorf("Error getting section ids for %s: %w", prefix, err) } if len(sectionIds) == 0 { @@ -89,10 +94,10 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo for _, sectionId := range sectionIds { content, err := scraper.getSectionContent(sectionId) if err != nil { - log.Fatalf("Error getting section content for section %s: %v", sectionId, err) + return fmt.Errorf("Error getting section content for section %s: %w", sectionId, err) } if err := scraper.writeSection(prefix, sectionId, content); err != nil { - log.Fatalf("Error writing section %s: %v", sectionId, err) + return fmt.Errorf("Error writing section %s: %w", sectionId, err) } time.Sleep(reqThrottle) } @@ -104,8 +109,10 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo log.Printf("[Scrape Complete] Finished scraping term %s in %v. Total sections %d: Total retries %d", term, time.Since(totalTime), scraper.totalScrapedSections, scraper.reqRetries) if err := scraper.validate(); err != nil { - log.Fatal("Validating failed: ", err) + return fmt.Errorf("Validating failed: %w", err) } + + return nil } type coursebookScraper struct { @@ -148,14 +155,14 @@ func newCoursebookScraper(term string, outDir string) *coursebookScraper { // lastCompletePrefix returns the last prefix (alphabetical order) that contains // html files for all of its section ids. returns an empty string if there are no // complete prefixes -func (s *coursebookScraper) lastCompletePrefix() string { +func (s *coursebookScraper) lastCompletePrefix() (string, error) { if err := s.ensureOutputFolder(); err != nil { - log.Fatal(err) + return "", fmt.Errorf("Error ensuring output folder: %w", err) } dir, err := os.ReadDir(filepath.Join(s.outDir, s.term)) if err != nil { - log.Fatalf("failed to read output directory: %v", err) + return "", fmt.Errorf("Error reading output directory: %w", err) } foundPrefixes := make([]string, 0, len(s.prefixes)) @@ -169,21 +176,21 @@ func (s *coursebookScraper) lastCompletePrefix() string { for _, prefix := range foundPrefixes { missing, err := s.getMissingIdsForPrefix(prefix) if err != nil { - log.Fatalf("Failed to get ids: %v", err) + return "", fmt.Errorf("Failed to get ids: %w", err) } if len(missing) == 0 { - return prefix + return prefix, nil } time.Sleep(reqThrottle) } - return "" + return "", nil } // ensurePrefixFolder creates {outDir}/term if it does not exist func (s *coursebookScraper) ensureOutputFolder() error { if err := os.MkdirAll(filepath.Join(s.outDir, s.term), 0755); err != nil { - return fmt.Errorf("failed to create term forlder: %w", err) + return fmt.Errorf("failed to create term folder: %w", err) } return nil } @@ -235,7 +242,7 @@ func (s *coursebookScraper) getMissingIdsForPrefix(prefix string) ([]string, err dir, err := os.ReadDir(path) if err != nil { - log.Panicf("Failed to access folder %s: %v", path, err) + return sectionIds, fmt.Errorf("Failed to access folder %s: %w", path, err) } foundIds := make(map[string]bool) @@ -285,7 +292,7 @@ func (s *coursebookScraper) req(queryStr string, retries int, reqName string) (s err := utils.Retry(func() error { req, err := http.NewRequest("POST", "https://coursebook.utdallas.edu/clips/clip-cb11-hat.zog", strings.NewReader(queryStr)) if err != nil { - log.Fatalf("Http request failed: %v", err) + return fmt.Errorf("Http request failed: %w", err) } req.Header = s.coursebookHeaders @@ -345,7 +352,7 @@ func (s *coursebookScraper) validate() error { log.Printf("[Validation] Missing %d sections for %s", len(ids), prefix) if err := s.ensurePrefixFolder(prefix); err != nil { - log.Fatal(err) + return fmt.Errorf("Error ensuring prefix folder: %w", err) } for _, id := range ids { From b22c4ee6a0fd3b18656330df434db57d1db40d93 Mon Sep 17 00:00:00 2001 From: Nitin Venkat Thota Date: Sun, 27 Sep 2026 02:45:06 -0500 Subject: [PATCH 4/7] Return errors from scrapers instead of panicking Scrapers previously called panic/log.Fatal/log.Panic on failure, which crashed the whole process and threw away progress on long-running scrapes (astra, coursebook). Scraper functions and the helpers they call (RefreshToken, RefreshAstraToken, GetCoursePrefixes, ExtractTextAndHref) now return errors instead, propagated up through main, which exits non-zero on failure. Fixes UTDNebula/api-tools#192 Co-Authored-By: Claude Sonnet 5 --- main.go | 31 +++++++++++------- parser/cometCalendarParser.go | 4 ++- scrapers/academicCalendars.go | 43 +++++++++++++++--------- scrapers/astra.go | 30 ++++++++++------- scrapers/budgets.go | 35 ++++++++++++-------- scrapers/cometCalendar.go | 11 +++---- scrapers/coursebook.go | 60 ++++++++++++++++++++++------------ scrapers/degrees.go | 11 ++++--- scrapers/discounts.go | 11 ++++--- scrapers/errors_test.go | 61 +++++++++++++++++++++++++++++++++++ scrapers/map.go | 22 +++++++------ scrapers/mazevo.go | 46 +++++++++++++++++++------- scrapers/profiles.go | 34 +++++++++++-------- utils/methods.go | 36 ++++++++++----------- utils/utils_test.go | 5 ++- 15 files changed, 296 insertions(+), 144 deletions(-) create mode 100644 scrapers/errors_test.go diff --git a/main.go b/main.go index 2789241..1228be1 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,10 @@ import ( ) func main() { + os.Exit(run()) +} + +func run() int { // Load environment variables godotenv.Load() @@ -121,31 +125,35 @@ func main() { case *scrape: switch { case *scrapeProfiles: - scrapers.ScrapeProfiles(*outDir) + err = scrapers.ScrapeProfiles(*outDir) case *scrapeCoursebook: if *term == "" { log.Panic("No term specified for coursebook scraping! Use -term to specify.") } - scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume) + err = scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume) case *scrapeDiscounts: - scrapers.ScrapeDiscounts(*outDir) + err = scrapers.ScrapeDiscounts(*outDir) case *cometCalendar: - scrapers.ScrapeCometCalendar(*outDir) + err = scrapers.ScrapeCometCalendar(*outDir) case *astra: - scrapers.ScrapeAstra(*outDir) + err = scrapers.ScrapeAstra(*outDir) case *mazevo: - scrapers.ScrapeMazevo(*outDir) + err = scrapers.ScrapeMazevo(*outDir) case *mapFlag: - scrapers.ScrapeMapLocations(*outDir) + err = scrapers.ScrapeMapLocations(*outDir) case *academicCalendars: - scrapers.ScrapeAcademicCalendars(*outDir) + err = scrapers.ScrapeAcademicCalendars(*outDir) case *degrees: - scrapers.ScrapeDegrees(*outDir) + err = scrapers.ScrapeDegrees(*outDir) case *budgets: - scrapers.ScrapeBudgets(*outDir) + err = scrapers.ScrapeBudgets(*outDir) default: log.Panic("You must specify which type of scraping you would like to perform with one of the scraping flags!") } + if err != nil { + log.Printf("Scraping failed: %v", err) + return 1 + } case *parse: switch { case *cometCalendar: @@ -186,6 +194,7 @@ func main() { } default: flag.PrintDefaults() - return + return 0 } + return 0 } diff --git a/parser/cometCalendarParser.go b/parser/cometCalendarParser.go index 70ca0ec..bca0a32 100644 --- a/parser/cometCalendarParser.go +++ b/parser/cometCalendarParser.go @@ -220,7 +220,9 @@ func getLocationAbbreviations(inDir string) (map[string]string, []string, error) if err != nil { if os.IsNotExist(err) { // Force scrape the locations if it doesn't exist. Get the map file again - scrapers.ScrapeMapLocations(inDir) + if err := scrapers.ScrapeMapLocations(inDir); err != nil { + return nil, nil, err + } ParseMapLocations(inDir, inDir) // If it fails to get the locations again, it's not because location is unscraped diff --git a/scrapers/academicCalendars.go b/scrapers/academicCalendars.go index df51b6d..5939f43 100644 --- a/scrapers/academicCalendars.go +++ b/scrapers/academicCalendars.go @@ -25,9 +25,10 @@ type AcademicCalendar struct { Time string } -func ScrapeAcademicCalendars(outDir string) { +func ScrapeAcademicCalendars(outDir string) error { // Start chromedp chromedpCtx, cancel := utils.InitChromeDp() + defer cancel() // Get sub folder from output folder outSubDir := filepath.Join(outDir, "academicCalendars") @@ -36,7 +37,7 @@ func ScrapeAcademicCalendars(outDir string) { os.RemoveAll(outSubDir) err := os.MkdirAll(outSubDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("create academic calendar output directory: %w", err) } // Go to listings page @@ -44,7 +45,7 @@ func ScrapeAcademicCalendars(outDir string) { chromedp.Navigate(`https://www.utdallas.edu/academics/calendar/`), ) if err != nil { - panic(err) + return fmt.Errorf("navigate to academic calendars: %w", err) } // Selector for the scraping the calendar nodes @@ -59,14 +60,14 @@ func ScrapeAcademicCalendars(outDir string) { chromedp.TextContent("h2.wp-block-heading", &academicCalendars[0].Title, chromedp.ByQuery), ) if err != nil { - panic(err) + return fmt.Errorf("read current calendar title: %w", err) } var currentNode []*cdp.Node err = chromedp.Run(chromedpCtx, chromedp.Nodes(currentSel, ¤tNode, chromedp.ByQuery), ) if err != nil { - panic(err) + return fmt.Errorf("read current calendar link: %w", err) } for i := 0; i < len(currentNode[0].Attributes); i += 2 { if currentNode[0].Attributes[i] == "href" { @@ -80,9 +81,12 @@ func ScrapeAcademicCalendars(outDir string) { chromedp.Nodes(futureSel, &futureNodes, chromedp.BySearch), ) if err != nil { - panic(err) + return fmt.Errorf("read future calendar links: %w", err) + } + links, err := utils.ExtractTextAndHref(futureNodes, chromedpCtx) + if err != nil { + return err } - links := utils.ExtractTextAndHref(futureNodes, chromedpCtx) for _, link := range links { academicCalendars = append(academicCalendars, AcademicCalendar{ Title: link.Text, @@ -97,9 +101,12 @@ func ScrapeAcademicCalendars(outDir string) { chromedp.Nodes(pastSel, &pastNodes, chromedp.BySearch), ) if err != nil { - panic(err) + return fmt.Errorf("read past calendar links: %w", err) + } + links, err = utils.ExtractTextAndHref(pastNodes, chromedpCtx) + if err != nil { + return err } - links = utils.ExtractTextAndHref(pastNodes, chromedpCtx) for _, link := range links { academicCalendars = append(academicCalendars, AcademicCalendar{ Title: link.Text, @@ -113,41 +120,45 @@ func ScrapeAcademicCalendars(outDir string) { // Download all PDFs for _, academicCalendar := range academicCalendars { - downloadPdfFromBox( + if err := downloadPdfFromBox( academicCalendar.Href, academicCalendar.Time+"-"+academicCalendar.Title, outSubDir, - ) + ); err != nil { + return err + } } + return nil } -func downloadPdfFromBox(href string, filename string, outDir string) { +func downloadPdfFromBox(href string, filename string, outDir string) error { // Create blank file out, err := os.Create(filepath.Join(outDir, fmt.Sprintf("%s.pdf", filename))) if err != nil { - panic(err) + return fmt.Errorf("create calendar PDF: %w", err) } defer out.Close() // Pull ID from link parsedLink, err := url.Parse(href) if err != nil { - panic(err) + return fmt.Errorf("parse calendar URL: %w", err) } fileId := path.Base(parsedLink.Path) // Use box download link with ID resp, err := http.Get(fmt.Sprintf("https://utdallas.box.com/shared/static/%s.pdf", fileId)) if err != nil { - panic(err) + return fmt.Errorf("download calendar PDF: %w", err) } defer resp.Body.Close() // Output response to blank file _, err = io.Copy(out, resp.Body) if err != nil { - panic(err) + return fmt.Errorf("save calendar PDF: %w", err) } log.Printf("Scraped academic calendar %s!", filename) + return nil } diff --git a/scrapers/astra.go b/scrapers/astra.go index 0eb0a15..d8d139b 100644 --- a/scrapers/astra.go +++ b/scrapers/astra.go @@ -20,14 +20,15 @@ import ( var MAX_EVENTS_PER_DAY = 5000 // ScrapeAstra iterates day-by-day through Astra events and persists the raw JSON output. -func ScrapeAstra(outDir string) { +func ScrapeAstra(outDir string) error { // Start chromedp chromedpCtx, cancel := utils.InitChromeDp() + defer cancel() // Make output folder err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return err } days := "{" // String JSON for storing results by day @@ -42,7 +43,10 @@ func ScrapeAstra(outDir string) { cli := &http.Client{Transport: tr} // Get cookies for auth - astraHeaders := utils.RefreshAstraToken(chromedpCtx) + astraHeaders, err := utils.RefreshAstraToken(chromedpCtx) + if err != nil { + return err + } time.Sleep(500 * time.Millisecond) cancel() // Don't need chromedp anymore @@ -60,27 +64,28 @@ func ScrapeAstra(outDir string) { url := fmt.Sprintf("https://www.aaiscloud.com/UTXDallas/~api/calendar/CalendarWeekGrid?_dc=%d&action=GET&start=0&limit=%d&isForWeekView=false&fields=ActivityId,ActivityPk,ActivityName,ParentActivityId,ParentActivityName,MeetingType,Description,StartDate,EndDate,DayOfWeek,StartMinute,EndMinute,ActivityTypeCode,ResourceId,CampusName,BuildingCode,RoomNumber,RoomName,LocationName,InstitutionId,SectionId,SectionPk,IsExam,IsCrosslist,IsAllDay,IsPrivate,EventId,EventPk,CurrentState,NotAllowedUsageMask,UsageColor,UsageColorIsPrimary,EventTypeColor,MaxAttendance,ActualAttendance,Capacity&filter=(StartDate%%3C%%3D%%22%sT23%%3A00%%3A00%%22)%%26%%26(EndDate%%3E%%3D%%22%sT00%%3A00%%3A00%%22)&page=1&sortOrder=%%2BStartDate,%%2BStartMinute", time.Now().UnixMilli(), MAX_EVENTS_PER_DAY, formattedDate, formattedDate) req, err := http.NewRequest("GET", url, nil) if err != nil { - panic(err) + return err } req.Header = astraHeaders res, err := cli.Do(req) if err != nil { - panic(err) + return err } if res.StatusCode != 200 { - log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status) + res.Body.Close() + return fmt.Errorf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status) } body, err := io.ReadAll(res.Body) + res.Body.Close() if err != nil { - panic(err) + return err } - res.Body.Close() stringBody := string(body) // Check for no events numEvents := fastjson.GetInt(body, "totalRecords") if numEvents >= MAX_EVENTS_PER_DAY { - log.Panic("ERROR: Max events per day exceeded!") + return fmt.Errorf("ERROR: Max events per day exceeded!") } // Add to record @@ -98,11 +103,12 @@ func ScrapeAstra(outDir string) { days = fmt.Sprintf("%s}", days) fptr, err := os.Create(fmt.Sprintf("%s/astraScraped.json", outDir)) if err != nil { - panic(err) + return err } + defer fptr.Close() _, err = fptr.Write([]byte(days)) if err != nil { - panic(err) + return err } - fptr.Close() + return nil } diff --git a/scrapers/budgets.go b/scrapers/budgets.go index dd6a0b1..81a372e 100644 --- a/scrapers/budgets.go +++ b/scrapers/budgets.go @@ -27,6 +27,7 @@ type Budget struct { func ScrapeBudgets(outDir string) error { // Start chromedp chromedpCtx, cancel := utils.InitChromeDp() + defer cancel() // Get sub folder from output folder outSubDir := filepath.Join(outDir, "budgets") @@ -35,7 +36,7 @@ func ScrapeBudgets(outDir string) error { os.RemoveAll(outSubDir) err := os.MkdirAll(outSubDir, 0777) if err != nil { - return fmt.Errorf("Error creating output folder: %w", err) + return fmt.Errorf("error creating output folder: %w", err) } // Go to listings page @@ -43,7 +44,7 @@ func ScrapeBudgets(outDir string) error { chromedp.Navigate(`https://finance.utdallas.edu/for-others/public-reports/`), ) if err != nil { - return fmt.Errorf("Error navigating to listings page: %w", err) + return fmt.Errorf("error navigating to listings page: %w", err) } // Selector for the scraping the budget nodes @@ -59,9 +60,12 @@ func ScrapeBudgets(outDir string) error { chromedp.Nodes(financialReportSel, &financialReportNodes, chromedp.BySearch), ) if err != nil { - return fmt.Errorf("Error retrieving Financial Report Nodes: %w", err) + return fmt.Errorf("error retrieving financial report nodes: %w", err) + } + links, err := utils.ExtractTextAndHref(financialReportNodes, chromedpCtx) + if err != nil { + return err } - links := utils.ExtractTextAndHref(financialReportNodes, chromedpCtx) for _, link := range links { budgets = append(budgets, Budget{ Title: link.Text, @@ -75,9 +79,12 @@ func ScrapeBudgets(outDir string) error { chromedp.Nodes(budgetReportSel, &budgetReportNodes, chromedp.BySearch), ) if err != nil { - return fmt.Errorf("Error retrieving Budget Report Nodes: %w", err) + return fmt.Errorf("error retrieving budget report nodes: %w", err) + } + links, err = utils.ExtractTextAndHref(budgetReportNodes, chromedpCtx) + if err != nil { + return err } - links = utils.ExtractTextAndHref(budgetReportNodes, chromedpCtx) for _, link := range links { budgets = append(budgets, Budget{ Title: link.Text, @@ -90,11 +97,13 @@ func ScrapeBudgets(outDir string) error { // Download all PDFs for _, budget := range budgets { - downloadPdf( + if err := downloadPdf( budget.Href, budget.Title, outSubDir, - ) + ); err != nil { + return err + } } return nil @@ -107,7 +116,7 @@ func downloadPdf(href string, filename string, outDir string) error { req, err := http.NewRequest("GET", href, nil) if err != nil { - return fmt.Errorf("Error creating request: %w", err) + return fmt.Errorf("error creating request: %w", err) } // Use a user agent and referer to avoid 599 errors @@ -116,7 +125,7 @@ func downloadPdf(href string, filename string, outDir string) error { resp, err := client.Do(req) if err != nil { - return fmt.Errorf("Error sending request: %w", err) + return fmt.Errorf("error sending request: %w", err) } defer resp.Body.Close() @@ -133,20 +142,20 @@ func downloadPdf(href string, filename string, outDir string) error { // Make output folder err = os.MkdirAll(outYearDir, 0777) if err != nil { - return fmt.Errorf("Error creating output folder: %w", err) + return fmt.Errorf("error creating output folder: %w", err) } // Create blank file out, err := os.Create(filepath.Join(outYearDir, fmt.Sprintf("%s.pdf", filename))) if err != nil { - return fmt.Errorf("Error creating blank file: %w", err) + return fmt.Errorf("error creating blank file: %w", err) } defer out.Close() // Output response to blank file _, err = io.Copy(out, resp.Body) if err != nil { - return fmt.Errorf("Error saving %s: %w", filename, err) + return fmt.Errorf("error saving %s: %w", filename, err) } log.Printf("Scraped budget %s!", filename) diff --git a/scrapers/cometCalendar.go b/scrapers/cometCalendar.go index 6080db9..094f952 100644 --- a/scrapers/cometCalendar.go +++ b/scrapers/cometCalendar.go @@ -72,7 +72,7 @@ type APICalendarResponse struct { func ScrapeCometCalendar(outDir string) error { err := os.MkdirAll(outDir, 0777) if err != nil { - return fmt.Errorf("Error creating output folder: %w", err) + return fmt.Errorf("error creating output folder: %w", err) } client := http.Client{ Timeout: 15 * time.Second, @@ -83,7 +83,7 @@ func ScrapeCometCalendar(outDir string) error { log.Printf("Getting the number of pages...") if err := callAndUnmarshal(&client, 0, &calendarData); err != nil { - return fmt.Errorf("Error fetching/decoding page %d: %w", 0, err) + return fmt.Errorf("error fetching/decoding page %d: %w", 0, err) } numPages := calendarData.Page["total"] log.Printf("The number of pages is %d!\n\n", numPages) @@ -92,13 +92,13 @@ func ScrapeCometCalendar(outDir string) error { for page := range numPages { log.Printf("Scraping events of page %d...", page+1) if err := callAndUnmarshal(&client, page+1, &calendarData); err != nil { - return fmt.Errorf("Error fetching/decoding page %d: %w", page+1, err) + return fmt.Errorf("error fetching/decoding page %d: %w", page+1, err) } for _, event := range calendarData.Events { // Parse all necessary info startTime, endTime, err := getTime(event.Event) if err != nil { - return fmt.Errorf("Error parsing start & end times: %w", err) + return fmt.Errorf("error parsing start & end times: %w", err) } if startTime.After(endTime) { fmt.Printf("start: %s, end: %s\n", startTime, endTime) @@ -132,11 +132,10 @@ func ScrapeCometCalendar(outDir string) error { writePath := fmt.Sprintf("%s/cometCalendarScraped.json", outDir) if err := utils.WriteJSON(writePath, calendarEvents); err != nil { - return fmt.Errorf("Error writing JSON to path: %w", err) + return fmt.Errorf("error writing JSON to path: %w", err) } log.Printf("Finished scraping %d events successfully!\n\n", len(calendarEvents)) - return nil } diff --git a/scrapers/coursebook.go b/scrapers/coursebook.go index 299b0ec..e794109 100644 --- a/scrapers/coursebook.go +++ b/scrapers/coursebook.go @@ -37,15 +37,17 @@ const ( // ScrapeCoursebook scrapes utd coursebook for the provided term (semester) func ScrapeCoursebook(term string, startPrefix string, outDir string, resume bool) error { - inputError := errors.New("InvalidInputError") if startPrefix != "" && !prefixRegex.MatchString(startPrefix) { - return fmt.Errorf("Invalid starting prefix %s, must match format cp_{abcde}: %w", startPrefix, inputError) + return fmt.Errorf("invalid starting prefix %s, must match format cp_{abcde}", startPrefix) } if !termRegex.MatchString(term) { - return fmt.Errorf("Invalid term %s, must match format {00-99}{s/f/u}: %w", term, inputError) + return fmt.Errorf("invalid term %s, must match format {00-99}{s/f/u}", term) } - scraper := newCoursebookScraper(term, outDir) + scraper, err := newCoursebookScraper(term, outDir) + if err != nil { + return err + } defer scraper.chromedpCancel() if resume && startPrefix == "" { @@ -53,7 +55,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo var err error startPrefix, err = scraper.lastCompletePrefix() if err != nil { - return fmt.Errorf("Error getting last complete prefix: %w", err) + return fmt.Errorf("error getting last complete prefix: %w", err) } } @@ -67,7 +69,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo start := time.Now() if err := scraper.ensurePrefixFolder(prefix); err != nil { - return fmt.Errorf("Error ensuring prefix folder: %w", err) + return fmt.Errorf("error ensuring prefix folder: %w", err) } var sectionIds []string @@ -81,7 +83,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo } if err != nil { - return fmt.Errorf("Error getting section ids for %s: %w", prefix, err) + return fmt.Errorf("error getting section ids for %s: %w", prefix, err) } if len(sectionIds) == 0 { @@ -109,9 +111,8 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo log.Printf("[Scrape Complete] Finished scraping term %s in %v. Total sections %d: Total retries %d", term, time.Since(totalTime), scraper.totalScrapedSections, scraper.reqRetries) if err := scraper.validate(); err != nil { - return fmt.Errorf("Validating failed: %w", err) + return fmt.Errorf("validating failed: %w", err) } - return nil } @@ -131,25 +132,34 @@ type coursebookScraper struct { totalScrapedSections int } -func newCoursebookScraper(term string, outDir string) *coursebookScraper { +func newCoursebookScraper(term string, outDir string) (*coursebookScraper, error) { ctx, cancel := utils.InitChromeDp() httpClient := &http.Client{ Timeout: httpTimeout, } //prefixes in alphabetical order for skip prefix flag - prefixes := utils.GetCoursePrefixes(ctx) + prefixes, err := utils.GetCoursePrefixes(ctx) + if err != nil { + cancel() + return nil, err + } sort.Strings(prefixes) + headers, err := utils.RefreshToken(ctx) + if err != nil { + cancel() + return nil, err + } return &coursebookScraper{ chromedpCtx: ctx, chromedpCancel: cancel, httpClient: httpClient, prefixes: prefixes, - coursebookHeaders: utils.RefreshToken(ctx), + coursebookHeaders: headers, term: term, outDir: outDir, prefixIdsCache: make(map[string][]string), - } + }, nil } // lastCompletePrefix returns the last prefix (alphabetical order) that contains @@ -157,12 +167,12 @@ func newCoursebookScraper(term string, outDir string) *coursebookScraper { // complete prefixes func (s *coursebookScraper) lastCompletePrefix() (string, error) { if err := s.ensureOutputFolder(); err != nil { - return "", fmt.Errorf("Error ensuring output folder: %w", err) + return "", fmt.Errorf("error ensuring output folder: %w", err) } dir, err := os.ReadDir(filepath.Join(s.outDir, s.term)) if err != nil { - return "", fmt.Errorf("Error reading output directory: %w", err) + return "", fmt.Errorf("failed to read output directory: %w", err) } foundPrefixes := make([]string, 0, len(s.prefixes)) @@ -242,7 +252,7 @@ func (s *coursebookScraper) getMissingIdsForPrefix(prefix string) ([]string, err dir, err := os.ReadDir(path) if err != nil { - return sectionIds, fmt.Errorf("Failed to access folder %s: %w", path, err) + return nil, fmt.Errorf("failed to access folder %s: %w", path, err) } foundIds := make(map[string]bool) @@ -289,7 +299,11 @@ func (s *coursebookScraper) getSectionIdsForPrefix(prefix string) ([]string, err // req utility function for making calling the coursebook api func (s *coursebookScraper) req(queryStr string, retries int, reqName string) (string, error) { var res *http.Response + var refreshErr error err := utils.Retry(func() error { + if refreshErr != nil { + return refreshErr + } req, err := http.NewRequest("POST", "https://coursebook.utdallas.edu/clips/clip-cb11-hat.zog", strings.NewReader(queryStr)) if err != nil { return fmt.Errorf("Http request failed: %w", err) @@ -317,7 +331,13 @@ func (s *coursebookScraper) req(queryStr string, retries int, reqName string) (s return err }, retries, func(numRetries int) { utils.VPrintf("[Request Retry] Attempt %d of %d for request %s", numRetries, retries, reqName) - s.coursebookHeaders = utils.RefreshToken(s.chromedpCtx) + if refreshErr != nil { + return + } + s.coursebookHeaders, refreshErr = utils.RefreshToken(s.chromedpCtx) + if refreshErr != nil { + return + } s.reqRetries++ //back off exponentially @@ -352,16 +372,16 @@ func (s *coursebookScraper) validate() error { log.Printf("[Validation] Missing %d sections for %s", len(ids), prefix) if err := s.ensurePrefixFolder(prefix); err != nil { - return fmt.Errorf("Error ensuring prefix folder: %w", err) + return fmt.Errorf("error ensuring prefix folder: %w", err) } for _, id := range ids { content, err := s.getSectionContent(id) if err != nil { - return fmt.Errorf("error getting section content for section %s: %v", id, err) + return fmt.Errorf("error getting section content for section %s: %w", id, err) } if err := s.writeSection(prefix, id, content); err != nil { - return fmt.Errorf("error writing section %s: %v", id, err) + return fmt.Errorf("error writing section %s: %w", id, err) } time.Sleep(reqThrottle) } diff --git a/scrapers/degrees.go b/scrapers/degrees.go index 7ac4684..6787211 100644 --- a/scrapers/degrees.go +++ b/scrapers/degrees.go @@ -12,11 +12,11 @@ import ( const degreesUrl = "https://academics.utdallas.edu/degrees/" -func ScrapeDegrees(outDir string) { +func ScrapeDegrees(outDir string) error { // Ensure output directory exists err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return err } chromedpCtx, cancel := utils.InitChromeDp() @@ -28,7 +28,7 @@ func ScrapeDegrees(outDir string) { chromedp.WaitVisible("article .col-sm-12", chromedp.ByQuery), ) if err != nil { - log.Panicf("failed to scrape: %v", err) + return fmt.Errorf("failed to scrape: %v", err) } // Wait for the article content to load @@ -37,15 +37,16 @@ func ScrapeDegrees(outDir string) { var html string err = chromedp.Run(chromedpCtx, chromedp.OuterHTML("article .col-sm-12", &html)) if err != nil { - log.Panicf("failed to scrape: %v", err) + return fmt.Errorf("failed to scrape: %v", err) } // Write raw HTML to file outPath := fmt.Sprintf("%s/degreesScraped.html", outDir) err = os.WriteFile(outPath, []byte(html), 0644) if err != nil { - panic(err) + return err } log.Printf("Scraped degrees successfully!\n") + return nil } diff --git a/scrapers/discounts.go b/scrapers/discounts.go index 9b9be00..279a9e1 100644 --- a/scrapers/discounts.go +++ b/scrapers/discounts.go @@ -17,11 +17,11 @@ import ( const discountUrl = "https://sg.utdallas.edu/discount/" // ScrapeDiscounts retrieves the discount programs page HTML and saves it. -func ScrapeDiscounts(outDir string) { +func ScrapeDiscounts(outDir string) error { // Ensure output directory exists err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return err } chromedpCtx, cancel := utils.InitChromeDp() @@ -33,7 +33,7 @@ func ScrapeDiscounts(outDir string) { chromedp.Navigate(discountUrl), chromedp.WaitReady("body", chromedp.ByQuery), ); err != nil { - panic(err) + return err } // Wait for the content to load @@ -42,14 +42,15 @@ func ScrapeDiscounts(outDir string) { // Get the HTML content var html string if err := chromedp.Run(chromedpCtx, chromedp.InnerHTML("body", &html)); err != nil { - panic(err) + return err } // Write raw HTML to file outPath := fmt.Sprintf("%s/discountsScraped.html", outDir) if err := os.WriteFile(outPath, []byte(html), 0644); err != nil { - panic(err) + return err } log.Printf("Scraped discounts successfully!\n") + return nil } diff --git a/scrapers/errors_test.go b/scrapers/errors_test.go new file mode 100644 index 0000000..34b7e77 --- /dev/null +++ b/scrapers/errors_test.go @@ -0,0 +1,61 @@ +package scrapers + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestScrapersReturnOutputDirectoryErrors(t *testing.T) { + blocked := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(blocked, []byte("existing data"), 0600); err != nil { + t.Fatal(err) + } + outDir := filepath.Join(blocked, "output") + scrapers := map[string]func(string) error{ + "discounts": ScrapeDiscounts, "map": ScrapeMapLocations, + "degrees": ScrapeDegrees, "academic calendars": ScrapeAcademicCalendars, + "budgets": ScrapeBudgets, "astra": ScrapeAstra, + "comet calendar": ScrapeCometCalendar, "mazevo": ScrapeMazevo, + "profiles": ScrapeProfiles, + } + for name, scrape := range scrapers { + t.Run(name, func(t *testing.T) { + err := scrape(outDir) + var pathErr *os.PathError + if !errors.As(err, &pathErr) { + t.Fatalf("expected filesystem error, got %v", err) + } + }) + } + content, err := os.ReadFile(blocked) + if err != nil || string(content) != "existing data" { + t.Fatalf("existing data changed: %q, %v", content, err) + } +} + +func TestCoursebookReturnsValidationErrors(t *testing.T) { + for _, args := range []struct{ term, prefix string }{{"26f", "invalid"}, {"invalid", ""}} { + if err := ScrapeCoursebook(args.term, args.prefix, t.TempDir(), false); err == nil { + t.Fatal("expected validation error") + } + } +} + +func TestCoursebookResumeReturnsDirectoryError(t *testing.T) { + blocked := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(blocked, nil, 0600); err != nil { + t.Fatal(err) + } + scraper := &coursebookScraper{outDir: blocked, term: "26f"} + if _, err := scraper.lastCompletePrefix(); err == nil { + t.Fatal("expected output directory error") + } +} + +func TestDownloadPDFReturnsInvalidURLError(t *testing.T) { + if err := downloadPdf(":invalid", "FY26 Budget", t.TempDir()); err == nil { + t.Fatal("expected request error") + } +} diff --git a/scrapers/map.go b/scrapers/map.go index c4f4008..49a136e 100644 --- a/scrapers/map.go +++ b/scrapers/map.go @@ -28,11 +28,11 @@ const START_URL string = "https://api.concept3d.com" const END_URL string = "/?map=" + UTD_MAP_ID + "&key=" + API_KEY // ScrapeMapLocations downloads Concept3D responses and writes raw map data to disk. -func ScrapeMapLocations(outDir string) { +func ScrapeMapLocations(outDir string) error { // Make output folder err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return err } // Init http client @@ -47,7 +47,7 @@ func ScrapeMapLocations(outDir string) { url := START_URL + "/locations" + END_URL req, err := http.NewRequest("GET", url, nil) if err != nil { - panic(err) + return err } req.Header = http.Header{ "Content-type": {"application/json"}, @@ -55,16 +55,17 @@ func ScrapeMapLocations(outDir string) { } res, err := cli.Do(req) if err != nil { - panic(err) + return err } if res.StatusCode != 200 { - log.Panicf("ERROR: Status was: %s", res.Status) + res.Body.Close() + return fmt.Errorf("ERROR: Status was: %s", res.Status) } body, err := io.ReadAll(res.Body) + res.Body.Close() if err != nil { - panic(err) + return err } - res.Body.Close() stringBody := string(body) log.Print("Scraped Map Locations!") @@ -72,11 +73,12 @@ func ScrapeMapLocations(outDir string) { // Write data to output file fptr, err := os.Create(fmt.Sprintf("%s/mapLocationsScraped.json", outDir)) if err != nil { - panic(err) + return err } + defer fptr.Close() _, err = fptr.Write([]byte(stringBody)) if err != nil { - panic(err) + return err } - fptr.Close() + return nil } diff --git a/scrapers/mazevo.go b/scrapers/mazevo.go index 8910679..42ed029 100644 --- a/scrapers/mazevo.go +++ b/scrapers/mazevo.go @@ -21,12 +21,12 @@ import ( ) // ScrapeMazevo pulls Mazevo calendar events via the public API and stores the raw response. -func ScrapeMazevo(outDir string) { +func ScrapeMazevo(outDir string) error { // Make output folder outDir = filepath.Join(outDir, "Mazevo") err := os.MkdirAll(outDir, 0777) if err != nil { - log.Panic(err) + return err } ctx, cancel := utils.InitChromeDp() @@ -37,6 +37,13 @@ func ScrapeMazevo(outDir string) { isPending := false receivedChan := make(chan struct{}) + listenerErrors := make(chan error, 1) + reportListenerError := func(err error) { + select { + case listenerErrors <- err: + default: + } + } chromedp.ListenTarget(ctx, func(ev any) { switch ev := ev.(type) { @@ -46,21 +53,25 @@ func ScrapeMazevo(outDir string) { rawPostData := ev.Request.PostDataEntries[0].Bytes decodedPostData, err := base64.StdEncoding.DecodeString(rawPostData) if err != nil { - log.Panic(err) + reportListenerError(err) + return } var data map[string]any err = json.Unmarshal(decodedPostData, &data) if err != nil { - log.Panic(err) + reportListenerError(err) + return } eventsStart, err = time.Parse(time.RFC3339, data["start"].(string)) if err != nil { - log.Panic(err) + reportListenerError(err) + return } eventsEnd, err = time.Parse(time.RFC3339, data["end"].(string)) if err != nil { - log.Panic(err) + reportListenerError(err) + return } // Check if end is 1 month after start @@ -77,7 +88,10 @@ func ScrapeMazevo(outDir string) { // Signal that response is finished loading if isPending && ev.RequestID == reqID { isPending = false - receivedChan <- struct{}{} + select { + case receivedChan <- struct{}{}: + case <-ctx.Done(): + } } } @@ -85,7 +99,13 @@ func ScrapeMazevo(outDir string) { scrapeLoop := func(ctx context.Context) error { // Wait until events have been received - <-receivedChan + select { + case <-receivedChan: + case err := <-listenerErrors: + return err + case <-ctx.Done(): + return ctx.Err() + } // Read Response (JSON) bodyBytes, err := network.GetResponseBody(reqID).Do(ctx) @@ -97,17 +117,18 @@ func ScrapeMazevo(outDir string) { // Write event data to output file fptr, err := os.Create(fmt.Sprintf("%s/%s.json", outDir, eventsStart.Format("2006-01"))) if err != nil { - log.Panic(err) + return err } + defer fptr.Close() _, err = fptr.Write(bodyBytes) if err != nil { - log.Panic(err) + return err } // Click next month err = chromedp.Click("[aria-label=\"Move to Next Month\"]", chromedp.NodeVisible).Do(ctx) if err != nil { - log.Panic(err) + return err } return nil } @@ -127,7 +148,8 @@ func ScrapeMazevo(outDir string) { }), ) if err != nil { - log.Panic(err) + return err } + return nil } diff --git a/scrapers/profiles.go b/scrapers/profiles.go index 59ec59d..5f008bc 100644 --- a/scrapers/profiles.go +++ b/scrapers/profiles.go @@ -97,7 +97,7 @@ func getNodeText(node *cdp.Node) string { return node.Children[0].NodeValue } -func scrapeProfessorLinks(chromedpCtx context.Context) []string { +func scrapeProfessorLinks(chromedpCtx context.Context) ([]string, error) { var pageLinks []*cdp.Node _, err := chromedp.RunResponse(chromedpCtx, chromedp.Navigate(BASE_URL+"1"), @@ -109,12 +109,12 @@ func scrapeProfessorLinks(chromedpCtx context.Context) []string { ), ) if err != nil { - panic(err) + return nil, err } numPages, err := strconv.Atoi(getNodeText(pageLinks[len(pageLinks)-2])) if err != nil { - panic(err) + return nil, err } professorLinks := make([]string, 0, numPages) @@ -135,28 +135,31 @@ func scrapeProfessorLinks(chromedpCtx context.Context) []string { ), ) if err != nil { - panic(err) + return nil, err } } - return professorLinks + return professorLinks, nil } // ScrapeProfiles navigates UTD profile listings and writes professor metadata to JSON. -func ScrapeProfiles(outDir string) { +func ScrapeProfiles(outDir string) error { chromedpCtx, cancel := utils.InitChromeDp() defer cancel() err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return err } var professors []schema.Professor log.Print("Scraping professor links...") - professorLinks := scrapeProfessorLinks(chromedpCtx) + professorLinks, err := scrapeProfessorLinks(chromedpCtx) + if err != nil { + return err + } log.Print("Scraped professor links!") for _, link := range professorLinks { @@ -176,7 +179,7 @@ func ScrapeProfiles(outDir string) { }), ) if err != nil { - panic(err) + return err } // Get the image uri @@ -215,7 +218,7 @@ func ScrapeProfiles(outDir string) { }), ) if err != nil { - panic(err) + return err } } @@ -269,7 +272,7 @@ func ScrapeProfiles(outDir string) { ), ) if err != nil { - panic(err) + return err } utils.VPrint("Parsing list...") @@ -296,10 +299,13 @@ func ScrapeProfiles(outDir string) { // Write professor data to output file fptr, err := os.Create(fmt.Sprintf("%s/profiles.json", outDir)) if err != nil { - panic(err) + return err } + defer fptr.Close() encoder := json.NewEncoder(fptr) encoder.SetIndent("", "\t") - encoder.Encode(professors) - fptr.Close() + if err := encoder.Encode(professors); err != nil { + return err + } + return nil } diff --git a/utils/methods.go b/utils/methods.go index addb000..92e8762 100644 --- a/utils/methods.go +++ b/utils/methods.go @@ -54,14 +54,14 @@ func InitChromeDp() (chromedpCtx context.Context, cancelFnc context.CancelFunc) } // RefreshToken logs into CourseBook and returns headers containing a fresh session token. -func RefreshToken(chromedpCtx context.Context) map[string][]string { +func RefreshToken(chromedpCtx context.Context) (map[string][]string, error) { netID, err := GetEnv("LOGIN_NETID") if err != nil { - panic(err) + return nil, err } password, err := GetEnv("LOGIN_PASSWORD") if err != nil { - panic(err) + return nil, err } delayedRetryCallback := func(numRetries int) { @@ -87,7 +87,7 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string { }, 3, delayedRetryCallback) if err != nil { - panic(err) + return nil, err } time.Sleep(250 * time.Millisecond) @@ -124,7 +124,7 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string { }, 3, delayedRetryCallback) if err != nil { - panic(err) + return nil, err } return map[string][]string{ @@ -135,19 +135,19 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string { "Content-Type": {"application/x-www-form-urlencoded"}, "Cookie": cookieStrs, "Connection": {"keep-alive"}, - } + }, nil } // RefreshAstraToken signs into Astra and returns headers containing authentication cookies. -func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { +func RefreshAstraToken(chromedpCtx context.Context) (map[string][]string, error) { // Get username and password username, err := GetEnv("LOGIN_ASTRA_USERNAME") if err != nil { - log.Panic("LOGIN_ASTRA_USERNAME is missing from .env!") + return nil, err } password, err := GetEnv("LOGIN_ASTRA_PASSWORD") if err != nil { - log.Panic("LOGIN_ASTRA_PASSWORD is missing from .env!") + return nil, err } // Sign in @@ -166,7 +166,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { chromedp.WaitVisible(`body`, chromedp.ByQuery), ) if err != nil { - panic(err) + return nil, err } time.Sleep(250 * time.Millisecond) @@ -194,7 +194,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { }), ) if err != nil { - panic(err) + return nil, err } // Return headers, copied from a request the actual site made @@ -212,7 +212,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { "Sec-Fetch-Site": {"none"}, "Sec-Fetch-User": {"?1"}, "Priority": {"u=0, i"}, - } + }, nil } // WriteJSON encodes data as indented JSON and writes it to filepath. @@ -288,7 +288,7 @@ func Retry(action func() error, maxRetries int, retryCallback func(numRetries in } // GetCoursePrefixes retrieves all course prefix values from CourseBook. -func GetCoursePrefixes(chromedpCtx context.Context) []string { +func GetCoursePrefixes(chromedpCtx context.Context) ([]string, error) { // Might need to refresh the token every time we get new course prefixes in the future // refreshToken(chromedpCtx) @@ -308,10 +308,10 @@ func GetCoursePrefixes(chromedpCtx context.Context) []string { ), ) if err != nil { - log.Panic(err) + return nil, err } log.Printf("Found the %d course prefixes!", len(coursePrefixes)) - return coursePrefixes + return coursePrefixes, nil } // ConvertFromInterface attempts to convert a value into the requested type and returns a pointer when successful. @@ -327,7 +327,7 @@ type LinkResult struct { Href string } -func ExtractTextAndHref(nodes []*cdp.Node, chromedpCtx context.Context) []LinkResult { +func ExtractTextAndHref(nodes []*cdp.Node, chromedpCtx context.Context) ([]LinkResult, error) { output := []LinkResult{} var err error @@ -345,10 +345,10 @@ func ExtractTextAndHref(nodes []*cdp.Node, chromedpCtx context.Context) []LinkRe chromedp.TextContent(fmt.Sprintf(`a[href="%s"]`, href), &text, chromedp.ByQuery), ) if err != nil { - panic(err) + return nil, err } output = append(output, LinkResult{text, href}) } - return output + return output, nil } diff --git a/utils/utils_test.go b/utils/utils_test.go index aeb5ba2..7d4d29c 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -40,7 +40,10 @@ func TestRefreshToken(t *testing.T) { ctx, cancel := InitChromeDp() defer cancel() // Try refreshing token - headers := RefreshToken(ctx) + headers, err := RefreshToken(ctx) + if err != nil { + t.Fatalf("Failed to refresh token: %v", err) + } // Make sure we successfully got a PTGSESSID cookie for _, cookie := range headers["Cookie"] { if strings.HasPrefix(cookie, "PTGSESSID") { From b18da14b3043d9fd2e6d83921f27d2c3ad91628f Mon Sep 17 00:00:00 2001 From: Nitin Venkat Thota Date: Sun, 27 Sep 2026 04:40:08 -0500 Subject: [PATCH 5/7] Fix staticcheck ST1005 violations in error strings CI's staticcheck step (ST1005) requires error strings to be lowercase and not end with punctuation. Fixes the 6 flagged spots in astra.go and coursebook.go from the panic-to-error refactor. Co-Authored-By: Claude Sonnet 5 --- scrapers/astra.go | 4 ++-- scrapers/coursebook.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapers/astra.go b/scrapers/astra.go index d8d139b..fad159a 100644 --- a/scrapers/astra.go +++ b/scrapers/astra.go @@ -73,7 +73,7 @@ func ScrapeAstra(outDir string) error { } if res.StatusCode != 200 { res.Body.Close() - return fmt.Errorf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status) + return fmt.Errorf("status was: %s (if 404, you've likely been IP ratelimited)", res.Status) } body, err := io.ReadAll(res.Body) res.Body.Close() @@ -85,7 +85,7 @@ func ScrapeAstra(outDir string) error { // Check for no events numEvents := fastjson.GetInt(body, "totalRecords") if numEvents >= MAX_EVENTS_PER_DAY { - return fmt.Errorf("ERROR: Max events per day exceeded!") + return fmt.Errorf("max events per day exceeded") } // Add to record diff --git a/scrapers/coursebook.go b/scrapers/coursebook.go index e794109..708762e 100644 --- a/scrapers/coursebook.go +++ b/scrapers/coursebook.go @@ -96,10 +96,10 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo for _, sectionId := range sectionIds { content, err := scraper.getSectionContent(sectionId) if err != nil { - return fmt.Errorf("Error getting section content for section %s: %w", sectionId, err) + return fmt.Errorf("error getting section content for section %s: %w", sectionId, err) } if err := scraper.writeSection(prefix, sectionId, content); err != nil { - return fmt.Errorf("Error writing section %s: %w", sectionId, err) + return fmt.Errorf("error writing section %s: %w", sectionId, err) } time.Sleep(reqThrottle) } @@ -186,7 +186,7 @@ func (s *coursebookScraper) lastCompletePrefix() (string, error) { for _, prefix := range foundPrefixes { missing, err := s.getMissingIdsForPrefix(prefix) if err != nil { - return "", fmt.Errorf("Failed to get ids: %w", err) + return "", fmt.Errorf("failed to get ids: %w", err) } if len(missing) == 0 { return prefix, nil @@ -306,7 +306,7 @@ func (s *coursebookScraper) req(queryStr string, retries int, reqName string) (s } req, err := http.NewRequest("POST", "https://coursebook.utdallas.edu/clips/clip-cb11-hat.zog", strings.NewReader(queryStr)) if err != nil { - return fmt.Errorf("Http request failed: %w", err) + return fmt.Errorf("http request failed: %w", err) } req.Header = s.coursebookHeaders From 1c986f2511ded2ac474c5c54811b1cac7824129a Mon Sep 17 00:00:00 2001 From: ThatRell Date: Wed, 14 Oct 2026 23:15:25 -0500 Subject: [PATCH 6/7] double checking and cleaning up code for testing --- main.go | 10 ++-------- scrapers/academicCalendars.go | 20 ++++++++++---------- scrapers/astra.go | 18 +++++++++--------- scrapers/budgets.go | 4 ++-- scrapers/cometCalendar.go | 2 +- scrapers/coursebook.go | 2 +- scrapers/degrees.go | 7 ++++--- scrapers/discounts.go | 8 ++++---- scrapers/map.go | 14 +++++++------- scrapers/mazevo.go | 20 +++++++++++--------- scrapers/profiles.go | 18 +++++++++--------- utils/methods.go | 16 ++++++++-------- 12 files changed, 68 insertions(+), 71 deletions(-) diff --git a/main.go b/main.go index 1228be1..9a073d2 100644 --- a/main.go +++ b/main.go @@ -16,10 +16,7 @@ import ( ) func main() { - os.Exit(run()) -} -func run() int { // Load environment variables godotenv.Load() @@ -106,7 +103,7 @@ func run() int { logFile, err := os.Create(fmt.Sprintf("./logs/%d-%d-%dT%d-%d-%d.log", month, day, year, hour, min, sec)) if err != nil { - log.Fatal(err) + log.Panicf("Error creating log file: %v", err) } defer logFile.Close() @@ -151,8 +148,7 @@ func run() int { log.Panic("You must specify which type of scraping you would like to perform with one of the scraping flags!") } if err != nil { - log.Printf("Scraping failed: %v", err) - return 1 + log.Panicf("Scraping failed: %v", err) } case *parse: switch { @@ -194,7 +190,5 @@ func run() int { } default: flag.PrintDefaults() - return 0 } - return 0 } diff --git a/scrapers/academicCalendars.go b/scrapers/academicCalendars.go index 5939f43..9e0a415 100644 --- a/scrapers/academicCalendars.go +++ b/scrapers/academicCalendars.go @@ -37,7 +37,7 @@ func ScrapeAcademicCalendars(outDir string) error { os.RemoveAll(outSubDir) err := os.MkdirAll(outSubDir, 0777) if err != nil { - return fmt.Errorf("create academic calendar output directory: %w", err) + return fmt.Errorf("error creating output folder: %w", err) } // Go to listings page @@ -45,7 +45,7 @@ func ScrapeAcademicCalendars(outDir string) error { chromedp.Navigate(`https://www.utdallas.edu/academics/calendar/`), ) if err != nil { - return fmt.Errorf("navigate to academic calendars: %w", err) + return fmt.Errorf("error navigating to \"https://www.utdallas.edu/academics/calendar/\": %w", err) } // Selector for the scraping the calendar nodes @@ -60,14 +60,14 @@ func ScrapeAcademicCalendars(outDir string) error { chromedp.TextContent("h2.wp-block-heading", &academicCalendars[0].Title, chromedp.ByQuery), ) if err != nil { - return fmt.Errorf("read current calendar title: %w", err) + return fmt.Errorf("error extracting calendar heading: %w", err) } var currentNode []*cdp.Node err = chromedp.Run(chromedpCtx, chromedp.Nodes(currentSel, ¤tNode, chromedp.ByQuery), ) if err != nil { - return fmt.Errorf("read current calendar link: %w", err) + return fmt.Errorf("error retrieving the full calendar link: %w", err) } for i := 0; i < len(currentNode[0].Attributes); i += 2 { if currentNode[0].Attributes[i] == "href" { @@ -81,7 +81,7 @@ func ScrapeAcademicCalendars(outDir string) error { chromedp.Nodes(futureSel, &futureNodes, chromedp.BySearch), ) if err != nil { - return fmt.Errorf("read future calendar links: %w", err) + return fmt.Errorf("error retrieving future term links: %w", err) } links, err := utils.ExtractTextAndHref(futureNodes, chromedpCtx) if err != nil { @@ -101,7 +101,7 @@ func ScrapeAcademicCalendars(outDir string) error { chromedp.Nodes(pastSel, &pastNodes, chromedp.BySearch), ) if err != nil { - return fmt.Errorf("read past calendar links: %w", err) + return fmt.Errorf("error retrieving past term links: %w", err) } links, err = utils.ExtractTextAndHref(pastNodes, chromedpCtx) if err != nil { @@ -135,28 +135,28 @@ func downloadPdfFromBox(href string, filename string, outDir string) error { // Create blank file out, err := os.Create(filepath.Join(outDir, fmt.Sprintf("%s.pdf", filename))) if err != nil { - return fmt.Errorf("create calendar PDF: %w", err) + return fmt.Errorf("error creating blank file: %w", err) } defer out.Close() // Pull ID from link parsedLink, err := url.Parse(href) if err != nil { - return fmt.Errorf("parse calendar URL: %w", err) + return fmt.Errorf("error parsing calendar link: %w", err) } fileId := path.Base(parsedLink.Path) // Use box download link with ID resp, err := http.Get(fmt.Sprintf("https://utdallas.box.com/shared/static/%s.pdf", fileId)) if err != nil { - return fmt.Errorf("download calendar PDF: %w", err) + return fmt.Errorf("error downloading from box link: %w", err) } defer resp.Body.Close() // Output response to blank file _, err = io.Copy(out, resp.Body) if err != nil { - return fmt.Errorf("save calendar PDF: %w", err) + return fmt.Errorf("error outputing to blank file: %w", err) } log.Printf("Scraped academic calendar %s!", filename) diff --git a/scrapers/astra.go b/scrapers/astra.go index fad159a..050f4d8 100644 --- a/scrapers/astra.go +++ b/scrapers/astra.go @@ -28,7 +28,7 @@ func ScrapeAstra(outDir string) error { // Make output folder err := os.MkdirAll(outDir, 0777) if err != nil { - return err + return fmt.Errorf("error creating output folder: %w", err) } days := "{" // String JSON for storing results by day @@ -45,7 +45,7 @@ func ScrapeAstra(outDir string) error { // Get cookies for auth astraHeaders, err := utils.RefreshAstraToken(chromedpCtx) if err != nil { - return err + return fmt.Errorf("error getting cookies for auth: %w", err) } time.Sleep(500 * time.Millisecond) cancel() // Don't need chromedp anymore @@ -64,28 +64,28 @@ func ScrapeAstra(outDir string) error { url := fmt.Sprintf("https://www.aaiscloud.com/UTXDallas/~api/calendar/CalendarWeekGrid?_dc=%d&action=GET&start=0&limit=%d&isForWeekView=false&fields=ActivityId,ActivityPk,ActivityName,ParentActivityId,ParentActivityName,MeetingType,Description,StartDate,EndDate,DayOfWeek,StartMinute,EndMinute,ActivityTypeCode,ResourceId,CampusName,BuildingCode,RoomNumber,RoomName,LocationName,InstitutionId,SectionId,SectionPk,IsExam,IsCrosslist,IsAllDay,IsPrivate,EventId,EventPk,CurrentState,NotAllowedUsageMask,UsageColor,UsageColorIsPrimary,EventTypeColor,MaxAttendance,ActualAttendance,Capacity&filter=(StartDate%%3C%%3D%%22%sT23%%3A00%%3A00%%22)%%26%%26(EndDate%%3E%%3D%%22%sT00%%3A00%%3A00%%22)&page=1&sortOrder=%%2BStartDate,%%2BStartMinute", time.Now().UnixMilli(), MAX_EVENTS_PER_DAY, formattedDate, formattedDate) req, err := http.NewRequest("GET", url, nil) if err != nil { - return err + return fmt.Errorf("error creating http request: %w", err) } req.Header = astraHeaders res, err := cli.Do(req) if err != nil { - return err + return fmt.Errorf("error sending request: %w", err) } if res.StatusCode != 200 { res.Body.Close() - return fmt.Errorf("status was: %s (if 404, you've likely been IP ratelimited)", res.Status) + return fmt.Errorf("error: status was: %s\nIf the status is 404, you've likely been IP ratelimited", res.Status) } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { - return err + return fmt.Errorf("error reading http response body: %w", err) } stringBody := string(body) // Check for no events numEvents := fastjson.GetInt(body, "totalRecords") if numEvents >= MAX_EVENTS_PER_DAY { - return fmt.Errorf("max events per day exceeded") + return fmt.Errorf("error: Max events per day exceeded") } // Add to record @@ -103,12 +103,12 @@ func ScrapeAstra(outDir string) error { days = fmt.Sprintf("%s}", days) fptr, err := os.Create(fmt.Sprintf("%s/astraScraped.json", outDir)) if err != nil { - return err + return fmt.Errorf("error creating output file: %w", err) } defer fptr.Close() _, err = fptr.Write([]byte(days)) if err != nil { - return err + return fmt.Errorf("error writing event data to output file: %w", err) } return nil } diff --git a/scrapers/budgets.go b/scrapers/budgets.go index 81a372e..21ba554 100644 --- a/scrapers/budgets.go +++ b/scrapers/budgets.go @@ -44,7 +44,7 @@ func ScrapeBudgets(outDir string) error { chromedp.Navigate(`https://finance.utdallas.edu/for-others/public-reports/`), ) if err != nil { - return fmt.Errorf("error navigating to listings page: %w", err) + return fmt.Errorf("error navigating to budget listings page: %w", err) } // Selector for the scraping the budget nodes @@ -116,7 +116,7 @@ func downloadPdf(href string, filename string, outDir string) error { req, err := http.NewRequest("GET", href, nil) if err != nil { - return fmt.Errorf("error creating request: %w", err) + return fmt.Errorf("error creating http request: %w", err) } // Use a user agent and referer to avoid 599 errors diff --git a/scrapers/cometCalendar.go b/scrapers/cometCalendar.go index 094f952..a5119c9 100644 --- a/scrapers/cometCalendar.go +++ b/scrapers/cometCalendar.go @@ -157,7 +157,7 @@ func callAndUnmarshal(client *http.Client, page int, data *APICalendarResponse) return err } if response != nil && response.StatusCode != 200 { - return fmt.Errorf("ERROR: Non-200 status is returned, %s", response.Status) + return fmt.Errorf("error: Non-200 status is returned, %s", response.Status) } defer response.Body.Close() diff --git a/scrapers/coursebook.go b/scrapers/coursebook.go index 708762e..2072a76 100644 --- a/scrapers/coursebook.go +++ b/scrapers/coursebook.go @@ -172,7 +172,7 @@ func (s *coursebookScraper) lastCompletePrefix() (string, error) { dir, err := os.ReadDir(filepath.Join(s.outDir, s.term)) if err != nil { - return "", fmt.Errorf("failed to read output directory: %w", err) + return "", fmt.Errorf("error reading output directory: %w", err) } foundPrefixes := make([]string, 0, len(s.prefixes)) diff --git a/scrapers/degrees.go b/scrapers/degrees.go index 6787211..9b1f6f4 100644 --- a/scrapers/degrees.go +++ b/scrapers/degrees.go @@ -16,7 +16,7 @@ func ScrapeDegrees(outDir string) error { // Ensure output directory exists err := os.MkdirAll(outDir, 0777) if err != nil { - return err + return fmt.Errorf("error creating output folder: %w", err) } chromedpCtx, cancel := utils.InitChromeDp() @@ -28,7 +28,7 @@ func ScrapeDegrees(outDir string) error { chromedp.WaitVisible("article .col-sm-12", chromedp.ByQuery), ) if err != nil { - return fmt.Errorf("failed to scrape: %v", err) + return fmt.Errorf("failed to scrape degrees: %v", err) } // Wait for the article content to load @@ -44,7 +44,8 @@ func ScrapeDegrees(outDir string) error { outPath := fmt.Sprintf("%s/degreesScraped.html", outDir) err = os.WriteFile(outPath, []byte(html), 0644) if err != nil { - return err + + return fmt.Errorf("error writing html to file: %w", err) } log.Printf("Scraped degrees successfully!\n") diff --git a/scrapers/discounts.go b/scrapers/discounts.go index 279a9e1..e963a65 100644 --- a/scrapers/discounts.go +++ b/scrapers/discounts.go @@ -21,7 +21,7 @@ func ScrapeDiscounts(outDir string) error { // Ensure output directory exists err := os.MkdirAll(outDir, 0777) if err != nil { - return err + return fmt.Errorf("error creating output folder: %w", err) } chromedpCtx, cancel := utils.InitChromeDp() @@ -33,7 +33,7 @@ func ScrapeDiscounts(outDir string) error { chromedp.Navigate(discountUrl), chromedp.WaitReady("body", chromedp.ByQuery), ); err != nil { - return err + return fmt.Errorf("error navigating to the discount page: %w", err) } // Wait for the content to load @@ -42,13 +42,13 @@ func ScrapeDiscounts(outDir string) error { // Get the HTML content var html string if err := chromedp.Run(chromedpCtx, chromedp.InnerHTML("body", &html)); err != nil { - return err + return fmt.Errorf("error getting the html content of the discount page; %w", err) } // Write raw HTML to file outPath := fmt.Sprintf("%s/discountsScraped.html", outDir) if err := os.WriteFile(outPath, []byte(html), 0644); err != nil { - return err + return fmt.Errorf("error writing html to file: %w", err) } log.Printf("Scraped discounts successfully!\n") diff --git a/scrapers/map.go b/scrapers/map.go index 49a136e..3279e49 100644 --- a/scrapers/map.go +++ b/scrapers/map.go @@ -32,7 +32,7 @@ func ScrapeMapLocations(outDir string) error { // Make output folder err := os.MkdirAll(outDir, 0777) if err != nil { - return err + return fmt.Errorf("error creating output folder: %w", err) } // Init http client @@ -47,7 +47,7 @@ func ScrapeMapLocations(outDir string) error { url := START_URL + "/locations" + END_URL req, err := http.NewRequest("GET", url, nil) if err != nil { - return err + return fmt.Errorf("error creating http request: %w", err) } req.Header = http.Header{ "Content-type": {"application/json"}, @@ -55,16 +55,16 @@ func ScrapeMapLocations(outDir string) error { } res, err := cli.Do(req) if err != nil { - return err + return fmt.Errorf("error sending request: %w", err) } if res.StatusCode != 200 { res.Body.Close() - return fmt.Errorf("ERROR: Status was: %s", res.Status) + return fmt.Errorf("error: Status was: %s", res.Status) } body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { - return err + return fmt.Errorf("error reading response body: %w", err) } stringBody := string(body) @@ -73,12 +73,12 @@ func ScrapeMapLocations(outDir string) error { // Write data to output file fptr, err := os.Create(fmt.Sprintf("%s/mapLocationsScraped.json", outDir)) if err != nil { - return err + return fmt.Errorf("error creating output file: %w", err) } defer fptr.Close() _, err = fptr.Write([]byte(stringBody)) if err != nil { - return err + return fmt.Errorf("error writing map data to json file: %w", err) } return nil } diff --git a/scrapers/mazevo.go b/scrapers/mazevo.go index 42ed029..754565f 100644 --- a/scrapers/mazevo.go +++ b/scrapers/mazevo.go @@ -26,7 +26,7 @@ func ScrapeMazevo(outDir string) error { outDir = filepath.Join(outDir, "Mazevo") err := os.MkdirAll(outDir, 0777) if err != nil { - return err + return fmt.Errorf("error creating output folder: %w", err) } ctx, cancel := utils.InitChromeDp() @@ -37,6 +37,8 @@ func ScrapeMazevo(outDir string) error { isPending := false receivedChan := make(chan struct{}) + + // Non-blocking error reporting listenerErrors := make(chan error, 1) reportListenerError := func(err error) { select { @@ -53,24 +55,24 @@ func ScrapeMazevo(outDir string) error { rawPostData := ev.Request.PostDataEntries[0].Bytes decodedPostData, err := base64.StdEncoding.DecodeString(rawPostData) if err != nil { - reportListenerError(err) + reportListenerError(fmt.Errorf("error decoding string: %w", err)) return } var data map[string]any err = json.Unmarshal(decodedPostData, &data) if err != nil { - reportListenerError(err) + reportListenerError(fmt.Errorf("error unmarshalling decoded data: %w", err)) return } eventsStart, err = time.Parse(time.RFC3339, data["start"].(string)) if err != nil { - reportListenerError(err) + reportListenerError(fmt.Errorf("error getting event start time: %w", err)) return } eventsEnd, err = time.Parse(time.RFC3339, data["end"].(string)) if err != nil { - reportListenerError(err) + reportListenerError(fmt.Errorf("error getting event end time: %w", err)) return } @@ -117,18 +119,18 @@ func ScrapeMazevo(outDir string) error { // Write event data to output file fptr, err := os.Create(fmt.Sprintf("%s/%s.json", outDir, eventsStart.Format("2006-01"))) if err != nil { - return err + return fmt.Errorf("error creating output file: %w", err) } defer fptr.Close() _, err = fptr.Write(bodyBytes) if err != nil { - return err + return fmt.Errorf("error writing to output file: %w", err) } // Click next month err = chromedp.Click("[aria-label=\"Move to Next Month\"]", chromedp.NodeVisible).Do(ctx) if err != nil { - return err + return fmt.Errorf("error clicking next month button: %w", err) } return nil } @@ -148,7 +150,7 @@ func ScrapeMazevo(outDir string) error { }), ) if err != nil { - return err + return fmt.Errorf("error scraping mazevo: %w", err) } return nil diff --git a/scrapers/profiles.go b/scrapers/profiles.go index 5f008bc..c6b74c2 100644 --- a/scrapers/profiles.go +++ b/scrapers/profiles.go @@ -109,12 +109,12 @@ func scrapeProfessorLinks(chromedpCtx context.Context) ([]string, error) { ), ) if err != nil { - return nil, err + return nil, fmt.Errorf("error getting page links: %w", err) } numPages, err := strconv.Atoi(getNodeText(pageLinks[len(pageLinks)-2])) if err != nil { - return nil, err + return nil, fmt.Errorf("error getting number of pages: %w", err) } professorLinks := make([]string, 0, numPages) @@ -135,7 +135,7 @@ func scrapeProfessorLinks(chromedpCtx context.Context) ([]string, error) { ), ) if err != nil { - return nil, err + return nil, fmt.Errorf("error scraping page %d: %w", curPage, err) } } @@ -150,7 +150,7 @@ func ScrapeProfiles(outDir string) error { err := os.MkdirAll(outDir, 0777) if err != nil { - return err + return fmt.Errorf("error creating output folder: %w", err) } var professors []schema.Professor @@ -179,7 +179,7 @@ func ScrapeProfiles(outDir string) error { }), ) if err != nil { - return err + return fmt.Errorf("error scraping professor name: %w", err) } // Get the image uri @@ -218,7 +218,7 @@ func ScrapeProfiles(outDir string) error { }), ) if err != nil { - return err + return fmt.Errorf("error scraping imageUri: %w", err) } } @@ -272,7 +272,7 @@ func ScrapeProfiles(outDir string) error { ), ) if err != nil { - return err + return fmt.Errorf("error scraping professor phone number and office location: %w", err) } utils.VPrint("Parsing list...") @@ -299,13 +299,13 @@ func ScrapeProfiles(outDir string) error { // Write professor data to output file fptr, err := os.Create(fmt.Sprintf("%s/profiles.json", outDir)) if err != nil { - return err + return fmt.Errorf("error creating output file: %w", err) } defer fptr.Close() encoder := json.NewEncoder(fptr) encoder.SetIndent("", "\t") if err := encoder.Encode(professors); err != nil { - return err + return fmt.Errorf("error encoding professor profile to JSON: %w", err) } return nil } diff --git a/utils/methods.go b/utils/methods.go index 92e8762..081ce5b 100644 --- a/utils/methods.go +++ b/utils/methods.go @@ -57,11 +57,11 @@ func InitChromeDp() (chromedpCtx context.Context, cancelFnc context.CancelFunc) func RefreshToken(chromedpCtx context.Context) (map[string][]string, error) { netID, err := GetEnv("LOGIN_NETID") if err != nil { - return nil, err + return nil, fmt.Errorf("error getting env variable LOGIN_NETID: %w", err) } password, err := GetEnv("LOGIN_PASSWORD") if err != nil { - return nil, err + return nil, fmt.Errorf("error getting env variable LOGIN_PASSWORD: %w", err) } delayedRetryCallback := func(numRetries int) { @@ -87,7 +87,7 @@ func RefreshToken(chromedpCtx context.Context) (map[string][]string, error) { }, 3, delayedRetryCallback) if err != nil { - return nil, err + return nil, fmt.Errorf("error logging into coursebook: %w", err) } time.Sleep(250 * time.Millisecond) @@ -143,11 +143,11 @@ func RefreshAstraToken(chromedpCtx context.Context) (map[string][]string, error) // Get username and password username, err := GetEnv("LOGIN_ASTRA_USERNAME") if err != nil { - return nil, err + return nil, fmt.Errorf("error getting env variable LOGIN_ASTRA_USERNAME: %w", err) } password, err := GetEnv("LOGIN_ASTRA_PASSWORD") if err != nil { - return nil, err + return nil, fmt.Errorf("error getting env variable LOGIN_ASTRA_PASSWORD: %w", err) } // Sign in @@ -166,7 +166,7 @@ func RefreshAstraToken(chromedpCtx context.Context) (map[string][]string, error) chromedp.WaitVisible(`body`, chromedp.ByQuery), ) if err != nil { - return nil, err + return nil, fmt.Errorf("error signing into Astra: %w", err) } time.Sleep(250 * time.Millisecond) @@ -308,7 +308,7 @@ func GetCoursePrefixes(chromedpCtx context.Context) ([]string, error) { ), ) if err != nil { - return nil, err + return nil, fmt.Errorf("error getting course prefixes: %w", err) } log.Printf("Found the %d course prefixes!", len(coursePrefixes)) return coursePrefixes, nil @@ -345,7 +345,7 @@ func ExtractTextAndHref(nodes []*cdp.Node, chromedpCtx context.Context) ([]LinkR chromedp.TextContent(fmt.Sprintf(`a[href="%s"]`, href), &text, chromedp.ByQuery), ) if err != nil { - return nil, err + return nil, fmt.Errorf("error getting inner text: %w", err) } output = append(output, LinkResult{text, href}) } From b557bd458201b6ecbaeb09c93a33c3b045f6a95e Mon Sep 17 00:00:00 2001 From: ThatRell Date: Mon, 19 Oct 2026 22:36:53 -0500 Subject: [PATCH 7/7] removed unnecessary tests --- scrapers/errors_test.go | 61 ----------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 scrapers/errors_test.go diff --git a/scrapers/errors_test.go b/scrapers/errors_test.go deleted file mode 100644 index 34b7e77..0000000 --- a/scrapers/errors_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package scrapers - -import ( - "errors" - "os" - "path/filepath" - "testing" -) - -func TestScrapersReturnOutputDirectoryErrors(t *testing.T) { - blocked := filepath.Join(t.TempDir(), "file") - if err := os.WriteFile(blocked, []byte("existing data"), 0600); err != nil { - t.Fatal(err) - } - outDir := filepath.Join(blocked, "output") - scrapers := map[string]func(string) error{ - "discounts": ScrapeDiscounts, "map": ScrapeMapLocations, - "degrees": ScrapeDegrees, "academic calendars": ScrapeAcademicCalendars, - "budgets": ScrapeBudgets, "astra": ScrapeAstra, - "comet calendar": ScrapeCometCalendar, "mazevo": ScrapeMazevo, - "profiles": ScrapeProfiles, - } - for name, scrape := range scrapers { - t.Run(name, func(t *testing.T) { - err := scrape(outDir) - var pathErr *os.PathError - if !errors.As(err, &pathErr) { - t.Fatalf("expected filesystem error, got %v", err) - } - }) - } - content, err := os.ReadFile(blocked) - if err != nil || string(content) != "existing data" { - t.Fatalf("existing data changed: %q, %v", content, err) - } -} - -func TestCoursebookReturnsValidationErrors(t *testing.T) { - for _, args := range []struct{ term, prefix string }{{"26f", "invalid"}, {"invalid", ""}} { - if err := ScrapeCoursebook(args.term, args.prefix, t.TempDir(), false); err == nil { - t.Fatal("expected validation error") - } - } -} - -func TestCoursebookResumeReturnsDirectoryError(t *testing.T) { - blocked := filepath.Join(t.TempDir(), "file") - if err := os.WriteFile(blocked, nil, 0600); err != nil { - t.Fatal(err) - } - scraper := &coursebookScraper{outDir: blocked, term: "26f"} - if _, err := scraper.lastCompletePrefix(); err == nil { - t.Fatal("expected output directory error") - } -} - -func TestDownloadPDFReturnsInvalidURLError(t *testing.T) { - if err := downloadPdf(":invalid", "FY26 Budget", t.TempDir()); err == nil { - t.Fatal("expected request error") - } -}