diff --git a/main.go b/main.go index 2789241..9a073d2 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( ) func main() { + // Load environment variables godotenv.Load() @@ -102,7 +103,7 @@ func main() { logFile, err := os.Create(fmt.Sprintf("./logs/%d-%d-%dT%d-%d-%d.log", month, day, year, hour, min, sec)) if err != nil { - log.Fatal(err) + log.Panicf("Error creating log file: %v", err) } defer logFile.Close() @@ -121,31 +122,34 @@ func main() { case *scrape: switch { case *scrapeProfiles: - scrapers.ScrapeProfiles(*outDir) + err = scrapers.ScrapeProfiles(*outDir) case *scrapeCoursebook: if *term == "" { log.Panic("No term specified for coursebook scraping! Use -term to specify.") } - scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume) + err = scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume) case *scrapeDiscounts: - scrapers.ScrapeDiscounts(*outDir) + err = scrapers.ScrapeDiscounts(*outDir) case *cometCalendar: - scrapers.ScrapeCometCalendar(*outDir) + err = scrapers.ScrapeCometCalendar(*outDir) case *astra: - scrapers.ScrapeAstra(*outDir) + err = scrapers.ScrapeAstra(*outDir) case *mazevo: - scrapers.ScrapeMazevo(*outDir) + err = scrapers.ScrapeMazevo(*outDir) case *mapFlag: - scrapers.ScrapeMapLocations(*outDir) + err = scrapers.ScrapeMapLocations(*outDir) case *academicCalendars: - scrapers.ScrapeAcademicCalendars(*outDir) + err = scrapers.ScrapeAcademicCalendars(*outDir) case *degrees: - scrapers.ScrapeDegrees(*outDir) + err = scrapers.ScrapeDegrees(*outDir) case *budgets: - scrapers.ScrapeBudgets(*outDir) + err = scrapers.ScrapeBudgets(*outDir) default: log.Panic("You must specify which type of scraping you would like to perform with one of the scraping flags!") } + if err != nil { + log.Panicf("Scraping failed: %v", err) + } case *parse: switch { case *cometCalendar: @@ -186,6 +190,5 @@ func main() { } default: flag.PrintDefaults() - return } } diff --git a/parser/cometCalendarParser.go b/parser/cometCalendarParser.go index 70ca0ec..bca0a32 100644 --- a/parser/cometCalendarParser.go +++ b/parser/cometCalendarParser.go @@ -220,7 +220,9 @@ func getLocationAbbreviations(inDir string) (map[string]string, []string, error) if err != nil { if os.IsNotExist(err) { // Force scrape the locations if it doesn't exist. Get the map file again - scrapers.ScrapeMapLocations(inDir) + if err := scrapers.ScrapeMapLocations(inDir); err != nil { + return nil, nil, err + } ParseMapLocations(inDir, inDir) // If it fails to get the locations again, it's not because location is unscraped diff --git a/scrapers/academicCalendars.go b/scrapers/academicCalendars.go index df51b6d..9e0a415 100644 --- a/scrapers/academicCalendars.go +++ b/scrapers/academicCalendars.go @@ -25,9 +25,10 @@ type AcademicCalendar struct { Time string } -func ScrapeAcademicCalendars(outDir string) { +func ScrapeAcademicCalendars(outDir string) error { // Start chromedp chromedpCtx, cancel := utils.InitChromeDp() + defer cancel() // Get sub folder from output folder outSubDir := filepath.Join(outDir, "academicCalendars") @@ -36,7 +37,7 @@ func ScrapeAcademicCalendars(outDir string) { os.RemoveAll(outSubDir) err := os.MkdirAll(outSubDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } // Go to listings page @@ -44,7 +45,7 @@ func ScrapeAcademicCalendars(outDir string) { chromedp.Navigate(`https://www.utdallas.edu/academics/calendar/`), ) if err != nil { - panic(err) + return fmt.Errorf("error navigating to \"https://www.utdallas.edu/academics/calendar/\": %w", err) } // Selector for the scraping the calendar nodes @@ -59,14 +60,14 @@ func ScrapeAcademicCalendars(outDir string) { chromedp.TextContent("h2.wp-block-heading", &academicCalendars[0].Title, chromedp.ByQuery), ) if err != nil { - panic(err) + return fmt.Errorf("error extracting calendar heading: %w", err) } var currentNode []*cdp.Node err = chromedp.Run(chromedpCtx, chromedp.Nodes(currentSel, ¤tNode, chromedp.ByQuery), ) if err != nil { - panic(err) + return fmt.Errorf("error retrieving the full calendar link: %w", err) } for i := 0; i < len(currentNode[0].Attributes); i += 2 { if currentNode[0].Attributes[i] == "href" { @@ -80,9 +81,12 @@ func ScrapeAcademicCalendars(outDir string) { chromedp.Nodes(futureSel, &futureNodes, chromedp.BySearch), ) if err != nil { - panic(err) + return fmt.Errorf("error retrieving future term links: %w", err) + } + links, err := utils.ExtractTextAndHref(futureNodes, chromedpCtx) + if err != nil { + return err } - links := utils.ExtractTextAndHref(futureNodes, chromedpCtx) for _, link := range links { academicCalendars = append(academicCalendars, AcademicCalendar{ Title: link.Text, @@ -97,9 +101,12 @@ func ScrapeAcademicCalendars(outDir string) { chromedp.Nodes(pastSel, &pastNodes, chromedp.BySearch), ) if err != nil { - panic(err) + return fmt.Errorf("error retrieving past term links: %w", err) + } + links, err = utils.ExtractTextAndHref(pastNodes, chromedpCtx) + if err != nil { + return err } - links = utils.ExtractTextAndHref(pastNodes, chromedpCtx) for _, link := range links { academicCalendars = append(academicCalendars, AcademicCalendar{ Title: link.Text, @@ -113,41 +120,45 @@ func ScrapeAcademicCalendars(outDir string) { // Download all PDFs for _, academicCalendar := range academicCalendars { - downloadPdfFromBox( + if err := downloadPdfFromBox( academicCalendar.Href, academicCalendar.Time+"-"+academicCalendar.Title, outSubDir, - ) + ); err != nil { + return err + } } + return nil } -func downloadPdfFromBox(href string, filename string, outDir string) { +func downloadPdfFromBox(href string, filename string, outDir string) error { // Create blank file out, err := os.Create(filepath.Join(outDir, fmt.Sprintf("%s.pdf", filename))) if err != nil { - panic(err) + return fmt.Errorf("error creating blank file: %w", err) } defer out.Close() // Pull ID from link parsedLink, err := url.Parse(href) if err != nil { - panic(err) + return fmt.Errorf("error parsing calendar link: %w", err) } fileId := path.Base(parsedLink.Path) // Use box download link with ID resp, err := http.Get(fmt.Sprintf("https://utdallas.box.com/shared/static/%s.pdf", fileId)) if err != nil { - panic(err) + return fmt.Errorf("error downloading from box link: %w", err) } defer resp.Body.Close() // Output response to blank file _, err = io.Copy(out, resp.Body) if err != nil { - panic(err) + return fmt.Errorf("error outputing to blank file: %w", err) } log.Printf("Scraped academic calendar %s!", filename) + return nil } diff --git a/scrapers/astra.go b/scrapers/astra.go index 0eb0a15..050f4d8 100644 --- a/scrapers/astra.go +++ b/scrapers/astra.go @@ -20,14 +20,15 @@ import ( var MAX_EVENTS_PER_DAY = 5000 // ScrapeAstra iterates day-by-day through Astra events and persists the raw JSON output. -func ScrapeAstra(outDir string) { +func ScrapeAstra(outDir string) error { // Start chromedp chromedpCtx, cancel := utils.InitChromeDp() + defer cancel() // Make output folder err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } days := "{" // String JSON for storing results by day @@ -42,7 +43,10 @@ func ScrapeAstra(outDir string) { cli := &http.Client{Transport: tr} // Get cookies for auth - astraHeaders := utils.RefreshAstraToken(chromedpCtx) + astraHeaders, err := utils.RefreshAstraToken(chromedpCtx) + if err != nil { + return fmt.Errorf("error getting cookies for auth: %w", err) + } time.Sleep(500 * time.Millisecond) cancel() // Don't need chromedp anymore @@ -60,27 +64,28 @@ func ScrapeAstra(outDir string) { url := fmt.Sprintf("https://www.aaiscloud.com/UTXDallas/~api/calendar/CalendarWeekGrid?_dc=%d&action=GET&start=0&limit=%d&isForWeekView=false&fields=ActivityId,ActivityPk,ActivityName,ParentActivityId,ParentActivityName,MeetingType,Description,StartDate,EndDate,DayOfWeek,StartMinute,EndMinute,ActivityTypeCode,ResourceId,CampusName,BuildingCode,RoomNumber,RoomName,LocationName,InstitutionId,SectionId,SectionPk,IsExam,IsCrosslist,IsAllDay,IsPrivate,EventId,EventPk,CurrentState,NotAllowedUsageMask,UsageColor,UsageColorIsPrimary,EventTypeColor,MaxAttendance,ActualAttendance,Capacity&filter=(StartDate%%3C%%3D%%22%sT23%%3A00%%3A00%%22)%%26%%26(EndDate%%3E%%3D%%22%sT00%%3A00%%3A00%%22)&page=1&sortOrder=%%2BStartDate,%%2BStartMinute", time.Now().UnixMilli(), MAX_EVENTS_PER_DAY, formattedDate, formattedDate) req, err := http.NewRequest("GET", url, nil) if err != nil { - panic(err) + return fmt.Errorf("error creating http request: %w", err) } req.Header = astraHeaders res, err := cli.Do(req) if err != nil { - panic(err) + return fmt.Errorf("error sending request: %w", err) } if res.StatusCode != 200 { - log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status) + res.Body.Close() + return fmt.Errorf("error: status was: %s\nIf the status is 404, you've likely been IP ratelimited", res.Status) } body, err := io.ReadAll(res.Body) + res.Body.Close() if err != nil { - panic(err) + return fmt.Errorf("error reading http response body: %w", err) } - res.Body.Close() stringBody := string(body) // Check for no events numEvents := fastjson.GetInt(body, "totalRecords") if numEvents >= MAX_EVENTS_PER_DAY { - log.Panic("ERROR: Max events per day exceeded!") + return fmt.Errorf("error: Max events per day exceeded") } // Add to record @@ -98,11 +103,12 @@ func ScrapeAstra(outDir string) { days = fmt.Sprintf("%s}", days) fptr, err := os.Create(fmt.Sprintf("%s/astraScraped.json", outDir)) if err != nil { - panic(err) + return fmt.Errorf("error creating output file: %w", err) } + defer fptr.Close() _, err = fptr.Write([]byte(days)) if err != nil { - panic(err) + return fmt.Errorf("error writing event data to output file: %w", err) } - fptr.Close() + return nil } diff --git a/scrapers/budgets.go b/scrapers/budgets.go index 6fde3f9..21ba554 100644 --- a/scrapers/budgets.go +++ b/scrapers/budgets.go @@ -24,9 +24,10 @@ type Budget struct { Href string } -func ScrapeBudgets(outDir string) { +func ScrapeBudgets(outDir string) error { // Start chromedp chromedpCtx, cancel := utils.InitChromeDp() + defer cancel() // Get sub folder from output folder outSubDir := filepath.Join(outDir, "budgets") @@ -35,7 +36,7 @@ func ScrapeBudgets(outDir string) { os.RemoveAll(outSubDir) err := os.MkdirAll(outSubDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } // Go to listings page @@ -43,7 +44,7 @@ func ScrapeBudgets(outDir string) { chromedp.Navigate(`https://finance.utdallas.edu/for-others/public-reports/`), ) if err != nil { - panic(err) + return fmt.Errorf("error navigating to budget listings page: %w", err) } // Selector for the scraping the budget nodes @@ -59,9 +60,12 @@ func ScrapeBudgets(outDir string) { chromedp.Nodes(financialReportSel, &financialReportNodes, chromedp.BySearch), ) if err != nil { - panic(err) + return fmt.Errorf("error retrieving financial report nodes: %w", err) + } + links, err := utils.ExtractTextAndHref(financialReportNodes, chromedpCtx) + if err != nil { + return err } - links := utils.ExtractTextAndHref(financialReportNodes, chromedpCtx) for _, link := range links { budgets = append(budgets, Budget{ Title: link.Text, @@ -75,9 +79,12 @@ func ScrapeBudgets(outDir string) { chromedp.Nodes(budgetReportSel, &budgetReportNodes, chromedp.BySearch), ) if err != nil { - panic(err) + return fmt.Errorf("error retrieving budget report nodes: %w", err) + } + links, err = utils.ExtractTextAndHref(budgetReportNodes, chromedpCtx) + if err != nil { + return err } - links = utils.ExtractTextAndHref(budgetReportNodes, chromedpCtx) for _, link := range links { budgets = append(budgets, Budget{ Title: link.Text, @@ -90,22 +97,26 @@ func ScrapeBudgets(outDir string) { // Download all PDFs for _, budget := range budgets { - downloadPdf( + if err := downloadPdf( budget.Href, budget.Title, outSubDir, - ) + ); err != nil { + return err + } } + + return nil } -func downloadPdf(href string, filename string, outDir string) { +func downloadPdf(href string, filename string, outDir string) error { client := &http.Client{ Timeout: 30 * time.Second, } req, err := http.NewRequest("GET", href, nil) if err != nil { - panic(err) + return fmt.Errorf("error creating http request: %w", err) } // Use a user agent and referer to avoid 599 errors @@ -114,13 +125,13 @@ func downloadPdf(href string, filename string, outDir string) { resp, err := client.Do(req) if err != nil { - panic(err) + return fmt.Errorf("error sending request: %w", err) } defer resp.Body.Close() // Check response if resp.StatusCode != http.StatusOK { - panic(fmt.Errorf("failed to download \"%s\": status code %d", filename, resp.StatusCode)) + return fmt.Errorf("failed to download \"%s\": status code %d", filename, resp.StatusCode) } // Get sub folder from output folder @@ -131,24 +142,25 @@ func downloadPdf(href string, filename string, outDir string) { // Make output folder err = os.MkdirAll(outYearDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } // Create blank file out, err := os.Create(filepath.Join(outYearDir, fmt.Sprintf("%s.pdf", filename))) if err != nil { - panic(err) + return fmt.Errorf("error creating blank file: %w", err) } defer out.Close() // Output response to blank file _, err = io.Copy(out, resp.Body) if err != nil { - log.Printf("Error saving %s: %v", filename, err) - return + return fmt.Errorf("error saving %s: %w", filename, err) } log.Printf("Scraped budget %s!", filename) time.Sleep(1 * time.Second) + + return nil } diff --git a/scrapers/cometCalendar.go b/scrapers/cometCalendar.go index a461b4e..a5119c9 100644 --- a/scrapers/cometCalendar.go +++ b/scrapers/cometCalendar.go @@ -69,10 +69,10 @@ type APICalendarResponse struct { } // ScrapeCometCalendar retrieves calendar events through the API -func ScrapeCometCalendar(outDir string) { +func ScrapeCometCalendar(outDir string) error { err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } client := http.Client{ Timeout: 15 * time.Second, @@ -83,7 +83,7 @@ func ScrapeCometCalendar(outDir string) { log.Printf("Getting the number of pages...") if err := callAndUnmarshal(&client, 0, &calendarData); err != nil { - panic(err) + return fmt.Errorf("error fetching/decoding page %d: %w", 0, err) } numPages := calendarData.Page["total"] log.Printf("The number of pages is %d!\n\n", numPages) @@ -92,13 +92,13 @@ func ScrapeCometCalendar(outDir string) { for page := range numPages { log.Printf("Scraping events of page %d...", page+1) if err := callAndUnmarshal(&client, page+1, &calendarData); err != nil { - panic(err) + return fmt.Errorf("error fetching/decoding page %d: %w", page+1, err) } for _, event := range calendarData.Events { // Parse all necessary info startTime, endTime, err := getTime(event.Event) if err != nil { - panic(err) + return fmt.Errorf("error parsing start & end times: %w", err) } if startTime.After(endTime) { fmt.Printf("start: %s, end: %s\n", startTime, endTime) @@ -132,10 +132,11 @@ func ScrapeCometCalendar(outDir string) { writePath := fmt.Sprintf("%s/cometCalendarScraped.json", outDir) if err := utils.WriteJSON(writePath, calendarEvents); err != nil { - panic(err) + return fmt.Errorf("error writing JSON to path: %w", err) } log.Printf("Finished scraping %d events successfully!\n\n", len(calendarEvents)) + return nil } // callAndUnmarshal fetches a calendar page and decodes it into data. @@ -156,7 +157,7 @@ func callAndUnmarshal(client *http.Client, page int, data *APICalendarResponse) return err } if response != nil && response.StatusCode != 200 { - return fmt.Errorf("ERROR: Non-200 status is returned, %s", response.Status) + return fmt.Errorf("error: Non-200 status is returned, %s", response.Status) } defer response.Body.Close() diff --git a/scrapers/coursebook.go b/scrapers/coursebook.go index 539b6a8..2072a76 100644 --- a/scrapers/coursebook.go +++ b/scrapers/coursebook.go @@ -36,20 +36,27 @@ const ( ) // ScrapeCoursebook scrapes utd coursebook for the provided term (semester) -func ScrapeCoursebook(term string, startPrefix string, outDir string, resume bool) { +func ScrapeCoursebook(term string, startPrefix string, outDir string, resume bool) error { if startPrefix != "" && !prefixRegex.MatchString(startPrefix) { - log.Fatalf("Invalid starting prefix %s, must match format cp_{abcde}", startPrefix) + return fmt.Errorf("invalid starting prefix %s, must match format cp_{abcde}", startPrefix) } if !termRegex.MatchString(term) { - log.Fatalf("Invalid term %s, must match format {00-99}{s/f/u}", term) + return fmt.Errorf("invalid term %s, must match format {00-99}{s/f/u}", term) } - scraper := newCoursebookScraper(term, outDir) + scraper, err := newCoursebookScraper(term, outDir) + if err != nil { + return err + } defer scraper.chromedpCancel() if resume && startPrefix == "" { // providing a starting prefix overrides the resume flag - startPrefix = scraper.lastCompletePrefix() + var err error + startPrefix, err = scraper.lastCompletePrefix() + if err != nil { + return fmt.Errorf("error getting last complete prefix: %w", err) + } } log.Printf("[Begin Scrape] Starting scrape for term %s with %d prefixes", term, len(scraper.prefixes)) @@ -62,7 +69,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo start := time.Now() if err := scraper.ensurePrefixFolder(prefix); err != nil { - log.Fatal(err) + return fmt.Errorf("error ensuring prefix folder: %w", err) } var sectionIds []string @@ -76,7 +83,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo } if err != nil { - log.Fatalf("Error getting section ids for %s ", prefix) + return fmt.Errorf("error getting section ids for %s: %w", prefix, err) } if len(sectionIds) == 0 { @@ -89,10 +96,10 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo for _, sectionId := range sectionIds { content, err := scraper.getSectionContent(sectionId) if err != nil { - log.Fatalf("Error getting section content for section %s: %v", sectionId, err) + return fmt.Errorf("error getting section content for section %s: %w", sectionId, err) } if err := scraper.writeSection(prefix, sectionId, content); err != nil { - log.Fatalf("Error writing section %s: %v", sectionId, err) + return fmt.Errorf("error writing section %s: %w", sectionId, err) } time.Sleep(reqThrottle) } @@ -104,8 +111,9 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string, resume boo log.Printf("[Scrape Complete] Finished scraping term %s in %v. Total sections %d: Total retries %d", term, time.Since(totalTime), scraper.totalScrapedSections, scraper.reqRetries) if err := scraper.validate(); err != nil { - log.Fatal("Validating failed: ", err) + return fmt.Errorf("validating failed: %w", err) } + return nil } type coursebookScraper struct { @@ -124,38 +132,47 @@ type coursebookScraper struct { totalScrapedSections int } -func newCoursebookScraper(term string, outDir string) *coursebookScraper { +func newCoursebookScraper(term string, outDir string) (*coursebookScraper, error) { ctx, cancel := utils.InitChromeDp() httpClient := &http.Client{ Timeout: httpTimeout, } //prefixes in alphabetical order for skip prefix flag - prefixes := utils.GetCoursePrefixes(ctx) + prefixes, err := utils.GetCoursePrefixes(ctx) + if err != nil { + cancel() + return nil, err + } sort.Strings(prefixes) + headers, err := utils.RefreshToken(ctx) + if err != nil { + cancel() + return nil, err + } return &coursebookScraper{ chromedpCtx: ctx, chromedpCancel: cancel, httpClient: httpClient, prefixes: prefixes, - coursebookHeaders: utils.RefreshToken(ctx), + coursebookHeaders: headers, term: term, outDir: outDir, prefixIdsCache: make(map[string][]string), - } + }, nil } // lastCompletePrefix returns the last prefix (alphabetical order) that contains // html files for all of its section ids. returns an empty string if there are no // complete prefixes -func (s *coursebookScraper) lastCompletePrefix() string { +func (s *coursebookScraper) lastCompletePrefix() (string, error) { if err := s.ensureOutputFolder(); err != nil { - log.Fatal(err) + return "", fmt.Errorf("error ensuring output folder: %w", err) } dir, err := os.ReadDir(filepath.Join(s.outDir, s.term)) if err != nil { - log.Fatalf("failed to read output directory: %v", err) + return "", fmt.Errorf("error reading output directory: %w", err) } foundPrefixes := make([]string, 0, len(s.prefixes)) @@ -169,21 +186,21 @@ func (s *coursebookScraper) lastCompletePrefix() string { for _, prefix := range foundPrefixes { missing, err := s.getMissingIdsForPrefix(prefix) if err != nil { - log.Fatalf("Failed to get ids: %v", err) + return "", fmt.Errorf("failed to get ids: %w", err) } if len(missing) == 0 { - return prefix + return prefix, nil } time.Sleep(reqThrottle) } - return "" + return "", nil } // ensurePrefixFolder creates {outDir}/term if it does not exist func (s *coursebookScraper) ensureOutputFolder() error { if err := os.MkdirAll(filepath.Join(s.outDir, s.term), 0755); err != nil { - return fmt.Errorf("failed to create term forlder: %w", err) + return fmt.Errorf("failed to create term folder: %w", err) } return nil } @@ -235,7 +252,7 @@ func (s *coursebookScraper) getMissingIdsForPrefix(prefix string) ([]string, err dir, err := os.ReadDir(path) if err != nil { - log.Panicf("Failed to access folder %s: %v", path, err) + return nil, fmt.Errorf("failed to access folder %s: %w", path, err) } foundIds := make(map[string]bool) @@ -282,10 +299,14 @@ func (s *coursebookScraper) getSectionIdsForPrefix(prefix string) ([]string, err // req utility function for making calling the coursebook api func (s *coursebookScraper) req(queryStr string, retries int, reqName string) (string, error) { var res *http.Response + var refreshErr error err := utils.Retry(func() error { + if refreshErr != nil { + return refreshErr + } req, err := http.NewRequest("POST", "https://coursebook.utdallas.edu/clips/clip-cb11-hat.zog", strings.NewReader(queryStr)) if err != nil { - log.Fatalf("Http request failed: %v", err) + return fmt.Errorf("http request failed: %w", err) } req.Header = s.coursebookHeaders @@ -310,7 +331,13 @@ func (s *coursebookScraper) req(queryStr string, retries int, reqName string) (s return err }, retries, func(numRetries int) { utils.VPrintf("[Request Retry] Attempt %d of %d for request %s", numRetries, retries, reqName) - s.coursebookHeaders = utils.RefreshToken(s.chromedpCtx) + if refreshErr != nil { + return + } + s.coursebookHeaders, refreshErr = utils.RefreshToken(s.chromedpCtx) + if refreshErr != nil { + return + } s.reqRetries++ //back off exponentially @@ -345,16 +372,16 @@ func (s *coursebookScraper) validate() error { log.Printf("[Validation] Missing %d sections for %s", len(ids), prefix) if err := s.ensurePrefixFolder(prefix); err != nil { - log.Fatal(err) + return fmt.Errorf("error ensuring prefix folder: %w", err) } for _, id := range ids { content, err := s.getSectionContent(id) if err != nil { - return fmt.Errorf("error getting section content for section %s: %v", id, err) + return fmt.Errorf("error getting section content for section %s: %w", id, err) } if err := s.writeSection(prefix, id, content); err != nil { - return fmt.Errorf("error writing section %s: %v", id, err) + return fmt.Errorf("error writing section %s: %w", id, err) } time.Sleep(reqThrottle) } diff --git a/scrapers/degrees.go b/scrapers/degrees.go index 7ac4684..9b1f6f4 100644 --- a/scrapers/degrees.go +++ b/scrapers/degrees.go @@ -12,11 +12,11 @@ import ( const degreesUrl = "https://academics.utdallas.edu/degrees/" -func ScrapeDegrees(outDir string) { +func ScrapeDegrees(outDir string) error { // Ensure output directory exists err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } chromedpCtx, cancel := utils.InitChromeDp() @@ -28,7 +28,7 @@ func ScrapeDegrees(outDir string) { chromedp.WaitVisible("article .col-sm-12", chromedp.ByQuery), ) if err != nil { - log.Panicf("failed to scrape: %v", err) + return fmt.Errorf("failed to scrape degrees: %v", err) } // Wait for the article content to load @@ -37,15 +37,17 @@ func ScrapeDegrees(outDir string) { var html string err = chromedp.Run(chromedpCtx, chromedp.OuterHTML("article .col-sm-12", &html)) if err != nil { - log.Panicf("failed to scrape: %v", err) + return fmt.Errorf("failed to scrape: %v", err) } // Write raw HTML to file outPath := fmt.Sprintf("%s/degreesScraped.html", outDir) err = os.WriteFile(outPath, []byte(html), 0644) if err != nil { - panic(err) + + return fmt.Errorf("error writing html to file: %w", err) } log.Printf("Scraped degrees successfully!\n") + return nil } diff --git a/scrapers/discounts.go b/scrapers/discounts.go index 9b9be00..e963a65 100644 --- a/scrapers/discounts.go +++ b/scrapers/discounts.go @@ -17,11 +17,11 @@ import ( const discountUrl = "https://sg.utdallas.edu/discount/" // ScrapeDiscounts retrieves the discount programs page HTML and saves it. -func ScrapeDiscounts(outDir string) { +func ScrapeDiscounts(outDir string) error { // Ensure output directory exists err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } chromedpCtx, cancel := utils.InitChromeDp() @@ -33,7 +33,7 @@ func ScrapeDiscounts(outDir string) { chromedp.Navigate(discountUrl), chromedp.WaitReady("body", chromedp.ByQuery), ); err != nil { - panic(err) + return fmt.Errorf("error navigating to the discount page: %w", err) } // Wait for the content to load @@ -42,14 +42,15 @@ func ScrapeDiscounts(outDir string) { // Get the HTML content var html string if err := chromedp.Run(chromedpCtx, chromedp.InnerHTML("body", &html)); err != nil { - panic(err) + return fmt.Errorf("error getting the html content of the discount page; %w", err) } // Write raw HTML to file outPath := fmt.Sprintf("%s/discountsScraped.html", outDir) if err := os.WriteFile(outPath, []byte(html), 0644); err != nil { - panic(err) + return fmt.Errorf("error writing html to file: %w", err) } log.Printf("Scraped discounts successfully!\n") + return nil } diff --git a/scrapers/map.go b/scrapers/map.go index c4f4008..3279e49 100644 --- a/scrapers/map.go +++ b/scrapers/map.go @@ -28,11 +28,11 @@ const START_URL string = "https://api.concept3d.com" const END_URL string = "/?map=" + UTD_MAP_ID + "&key=" + API_KEY // ScrapeMapLocations downloads Concept3D responses and writes raw map data to disk. -func ScrapeMapLocations(outDir string) { +func ScrapeMapLocations(outDir string) error { // Make output folder err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } // Init http client @@ -47,7 +47,7 @@ func ScrapeMapLocations(outDir string) { url := START_URL + "/locations" + END_URL req, err := http.NewRequest("GET", url, nil) if err != nil { - panic(err) + return fmt.Errorf("error creating http request: %w", err) } req.Header = http.Header{ "Content-type": {"application/json"}, @@ -55,16 +55,17 @@ func ScrapeMapLocations(outDir string) { } res, err := cli.Do(req) if err != nil { - panic(err) + return fmt.Errorf("error sending request: %w", err) } if res.StatusCode != 200 { - log.Panicf("ERROR: Status was: %s", res.Status) + res.Body.Close() + return fmt.Errorf("error: Status was: %s", res.Status) } body, err := io.ReadAll(res.Body) + res.Body.Close() if err != nil { - panic(err) + return fmt.Errorf("error reading response body: %w", err) } - res.Body.Close() stringBody := string(body) log.Print("Scraped Map Locations!") @@ -72,11 +73,12 @@ func ScrapeMapLocations(outDir string) { // Write data to output file fptr, err := os.Create(fmt.Sprintf("%s/mapLocationsScraped.json", outDir)) if err != nil { - panic(err) + return fmt.Errorf("error creating output file: %w", err) } + defer fptr.Close() _, err = fptr.Write([]byte(stringBody)) if err != nil { - panic(err) + return fmt.Errorf("error writing map data to json file: %w", err) } - fptr.Close() + return nil } diff --git a/scrapers/mazevo.go b/scrapers/mazevo.go index 8910679..754565f 100644 --- a/scrapers/mazevo.go +++ b/scrapers/mazevo.go @@ -21,12 +21,12 @@ import ( ) // ScrapeMazevo pulls Mazevo calendar events via the public API and stores the raw response. -func ScrapeMazevo(outDir string) { +func ScrapeMazevo(outDir string) error { // Make output folder outDir = filepath.Join(outDir, "Mazevo") err := os.MkdirAll(outDir, 0777) if err != nil { - log.Panic(err) + return fmt.Errorf("error creating output folder: %w", err) } ctx, cancel := utils.InitChromeDp() @@ -38,6 +38,15 @@ func ScrapeMazevo(outDir string) { isPending := false receivedChan := make(chan struct{}) + // Non-blocking error reporting + listenerErrors := make(chan error, 1) + reportListenerError := func(err error) { + select { + case listenerErrors <- err: + default: + } + } + chromedp.ListenTarget(ctx, func(ev any) { switch ev := ev.(type) { case *network.EventRequestWillBeSent: @@ -46,21 +55,25 @@ func ScrapeMazevo(outDir string) { rawPostData := ev.Request.PostDataEntries[0].Bytes decodedPostData, err := base64.StdEncoding.DecodeString(rawPostData) if err != nil { - log.Panic(err) + reportListenerError(fmt.Errorf("error decoding string: %w", err)) + return } var data map[string]any err = json.Unmarshal(decodedPostData, &data) if err != nil { - log.Panic(err) + reportListenerError(fmt.Errorf("error unmarshalling decoded data: %w", err)) + return } eventsStart, err = time.Parse(time.RFC3339, data["start"].(string)) if err != nil { - log.Panic(err) + reportListenerError(fmt.Errorf("error getting event start time: %w", err)) + return } eventsEnd, err = time.Parse(time.RFC3339, data["end"].(string)) if err != nil { - log.Panic(err) + reportListenerError(fmt.Errorf("error getting event end time: %w", err)) + return } // Check if end is 1 month after start @@ -77,7 +90,10 @@ func ScrapeMazevo(outDir string) { // Signal that response is finished loading if isPending && ev.RequestID == reqID { isPending = false - receivedChan <- struct{}{} + select { + case receivedChan <- struct{}{}: + case <-ctx.Done(): + } } } @@ -85,7 +101,13 @@ func ScrapeMazevo(outDir string) { scrapeLoop := func(ctx context.Context) error { // Wait until events have been received - <-receivedChan + select { + case <-receivedChan: + case err := <-listenerErrors: + return err + case <-ctx.Done(): + return ctx.Err() + } // Read Response (JSON) bodyBytes, err := network.GetResponseBody(reqID).Do(ctx) @@ -97,17 +119,18 @@ func ScrapeMazevo(outDir string) { // Write event data to output file fptr, err := os.Create(fmt.Sprintf("%s/%s.json", outDir, eventsStart.Format("2006-01"))) if err != nil { - log.Panic(err) + return fmt.Errorf("error creating output file: %w", err) } + defer fptr.Close() _, err = fptr.Write(bodyBytes) if err != nil { - log.Panic(err) + return fmt.Errorf("error writing to output file: %w", err) } // Click next month err = chromedp.Click("[aria-label=\"Move to Next Month\"]", chromedp.NodeVisible).Do(ctx) if err != nil { - log.Panic(err) + return fmt.Errorf("error clicking next month button: %w", err) } return nil } @@ -127,7 +150,8 @@ func ScrapeMazevo(outDir string) { }), ) if err != nil { - log.Panic(err) + return fmt.Errorf("error scraping mazevo: %w", err) } + return nil } diff --git a/scrapers/profiles.go b/scrapers/profiles.go index 59ec59d..c6b74c2 100644 --- a/scrapers/profiles.go +++ b/scrapers/profiles.go @@ -97,7 +97,7 @@ func getNodeText(node *cdp.Node) string { return node.Children[0].NodeValue } -func scrapeProfessorLinks(chromedpCtx context.Context) []string { +func scrapeProfessorLinks(chromedpCtx context.Context) ([]string, error) { var pageLinks []*cdp.Node _, err := chromedp.RunResponse(chromedpCtx, chromedp.Navigate(BASE_URL+"1"), @@ -109,12 +109,12 @@ func scrapeProfessorLinks(chromedpCtx context.Context) []string { ), ) if err != nil { - panic(err) + return nil, fmt.Errorf("error getting page links: %w", err) } numPages, err := strconv.Atoi(getNodeText(pageLinks[len(pageLinks)-2])) if err != nil { - panic(err) + return nil, fmt.Errorf("error getting number of pages: %w", err) } professorLinks := make([]string, 0, numPages) @@ -135,28 +135,31 @@ func scrapeProfessorLinks(chromedpCtx context.Context) []string { ), ) if err != nil { - panic(err) + return nil, fmt.Errorf("error scraping page %d: %w", curPage, err) } } - return professorLinks + return professorLinks, nil } // ScrapeProfiles navigates UTD profile listings and writes professor metadata to JSON. -func ScrapeProfiles(outDir string) { +func ScrapeProfiles(outDir string) error { chromedpCtx, cancel := utils.InitChromeDp() defer cancel() err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + return fmt.Errorf("error creating output folder: %w", err) } var professors []schema.Professor log.Print("Scraping professor links...") - professorLinks := scrapeProfessorLinks(chromedpCtx) + professorLinks, err := scrapeProfessorLinks(chromedpCtx) + if err != nil { + return err + } log.Print("Scraped professor links!") for _, link := range professorLinks { @@ -176,7 +179,7 @@ func ScrapeProfiles(outDir string) { }), ) if err != nil { - panic(err) + return fmt.Errorf("error scraping professor name: %w", err) } // Get the image uri @@ -215,7 +218,7 @@ func ScrapeProfiles(outDir string) { }), ) if err != nil { - panic(err) + return fmt.Errorf("error scraping imageUri: %w", err) } } @@ -269,7 +272,7 @@ func ScrapeProfiles(outDir string) { ), ) if err != nil { - panic(err) + return fmt.Errorf("error scraping professor phone number and office location: %w", err) } utils.VPrint("Parsing list...") @@ -296,10 +299,13 @@ func ScrapeProfiles(outDir string) { // Write professor data to output file fptr, err := os.Create(fmt.Sprintf("%s/profiles.json", outDir)) if err != nil { - panic(err) + return fmt.Errorf("error creating output file: %w", err) } + defer fptr.Close() encoder := json.NewEncoder(fptr) encoder.SetIndent("", "\t") - encoder.Encode(professors) - fptr.Close() + if err := encoder.Encode(professors); err != nil { + return fmt.Errorf("error encoding professor profile to JSON: %w", err) + } + return nil } diff --git a/utils/methods.go b/utils/methods.go index addb000..081ce5b 100644 --- a/utils/methods.go +++ b/utils/methods.go @@ -54,14 +54,14 @@ func InitChromeDp() (chromedpCtx context.Context, cancelFnc context.CancelFunc) } // RefreshToken logs into CourseBook and returns headers containing a fresh session token. -func RefreshToken(chromedpCtx context.Context) map[string][]string { +func RefreshToken(chromedpCtx context.Context) (map[string][]string, error) { netID, err := GetEnv("LOGIN_NETID") if err != nil { - panic(err) + return nil, fmt.Errorf("error getting env variable LOGIN_NETID: %w", err) } password, err := GetEnv("LOGIN_PASSWORD") if err != nil { - panic(err) + return nil, fmt.Errorf("error getting env variable LOGIN_PASSWORD: %w", err) } delayedRetryCallback := func(numRetries int) { @@ -87,7 +87,7 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string { }, 3, delayedRetryCallback) if err != nil { - panic(err) + return nil, fmt.Errorf("error logging into coursebook: %w", err) } time.Sleep(250 * time.Millisecond) @@ -124,7 +124,7 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string { }, 3, delayedRetryCallback) if err != nil { - panic(err) + return nil, err } return map[string][]string{ @@ -135,19 +135,19 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string { "Content-Type": {"application/x-www-form-urlencoded"}, "Cookie": cookieStrs, "Connection": {"keep-alive"}, - } + }, nil } // RefreshAstraToken signs into Astra and returns headers containing authentication cookies. -func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { +func RefreshAstraToken(chromedpCtx context.Context) (map[string][]string, error) { // Get username and password username, err := GetEnv("LOGIN_ASTRA_USERNAME") if err != nil { - log.Panic("LOGIN_ASTRA_USERNAME is missing from .env!") + return nil, fmt.Errorf("error getting env variable LOGIN_ASTRA_USERNAME: %w", err) } password, err := GetEnv("LOGIN_ASTRA_PASSWORD") if err != nil { - log.Panic("LOGIN_ASTRA_PASSWORD is missing from .env!") + return nil, fmt.Errorf("error getting env variable LOGIN_ASTRA_PASSWORD: %w", err) } // Sign in @@ -166,7 +166,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { chromedp.WaitVisible(`body`, chromedp.ByQuery), ) if err != nil { - panic(err) + return nil, fmt.Errorf("error signing into Astra: %w", err) } time.Sleep(250 * time.Millisecond) @@ -194,7 +194,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { }), ) if err != nil { - panic(err) + return nil, err } // Return headers, copied from a request the actual site made @@ -212,7 +212,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string { "Sec-Fetch-Site": {"none"}, "Sec-Fetch-User": {"?1"}, "Priority": {"u=0, i"}, - } + }, nil } // WriteJSON encodes data as indented JSON and writes it to filepath. @@ -288,7 +288,7 @@ func Retry(action func() error, maxRetries int, retryCallback func(numRetries in } // GetCoursePrefixes retrieves all course prefix values from CourseBook. -func GetCoursePrefixes(chromedpCtx context.Context) []string { +func GetCoursePrefixes(chromedpCtx context.Context) ([]string, error) { // Might need to refresh the token every time we get new course prefixes in the future // refreshToken(chromedpCtx) @@ -308,10 +308,10 @@ func GetCoursePrefixes(chromedpCtx context.Context) []string { ), ) if err != nil { - log.Panic(err) + return nil, fmt.Errorf("error getting course prefixes: %w", err) } log.Printf("Found the %d course prefixes!", len(coursePrefixes)) - return coursePrefixes + return coursePrefixes, nil } // ConvertFromInterface attempts to convert a value into the requested type and returns a pointer when successful. @@ -327,7 +327,7 @@ type LinkResult struct { Href string } -func ExtractTextAndHref(nodes []*cdp.Node, chromedpCtx context.Context) []LinkResult { +func ExtractTextAndHref(nodes []*cdp.Node, chromedpCtx context.Context) ([]LinkResult, error) { output := []LinkResult{} var err error @@ -345,10 +345,10 @@ func ExtractTextAndHref(nodes []*cdp.Node, chromedpCtx context.Context) []LinkRe chromedp.TextContent(fmt.Sprintf(`a[href="%s"]`, href), &text, chromedp.ByQuery), ) if err != nil { - panic(err) + return nil, fmt.Errorf("error getting inner text: %w", err) } output = append(output, LinkResult{text, href}) } - return output + return output, nil } diff --git a/utils/utils_test.go b/utils/utils_test.go index aeb5ba2..7d4d29c 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -40,7 +40,10 @@ func TestRefreshToken(t *testing.T) { ctx, cancel := InitChromeDp() defer cancel() // Try refreshing token - headers := RefreshToken(ctx) + headers, err := RefreshToken(ctx) + if err != nil { + t.Fatalf("Failed to refresh token: %v", err) + } // Make sure we successfully got a PTGSESSID cookie for _, cookie := range headers["Cookie"] { if strings.HasPrefix(cookie, "PTGSESSID") {