192 - Return errors from scrapers instead of panicking

main.go

@@ -16,6 +16,7 @@ import (
 )
 
 func main() {
+
 	// Load environment variables
 	godotenv.Load()
 
@@ -102,7 +103,7 @@ func main() {
 	logFile, err := os.Create(fmt.Sprintf("./logs/%d-%d-%dT%d-%d-%d.log", month, day, year, hour, min, sec))
 
 	if err != nil {
-		log.Fatal(err)
+		log.Panicf("Error creating log file: %v", err)
 	}
 
 	defer logFile.Close()
@@ -121,31 +122,34 @@ func main() {
 	case *scrape:
 		switch {
 		case *scrapeProfiles:
-			scrapers.ScrapeProfiles(*outDir)
+			err = scrapers.ScrapeProfiles(*outDir)
 		case *scrapeCoursebook:
 			if *term == "" {
 				log.Panic("No term specified for coursebook scraping! Use -term to specify.")
 			}
-			scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume)
+			err = scrapers.ScrapeCoursebook(*term, *startPrefix, *outDir, *resume)
 		case *scrapeDiscounts:
-			scrapers.ScrapeDiscounts(*outDir)
+			err = scrapers.ScrapeDiscounts(*outDir)
 		case *cometCalendar:
-			scrapers.ScrapeCometCalendar(*outDir)
+			err = scrapers.ScrapeCometCalendar(*outDir)
 		case *astra:
-			scrapers.ScrapeAstra(*outDir)
+			err = scrapers.ScrapeAstra(*outDir)
 		case *mazevo:
-			scrapers.ScrapeMazevo(*outDir)
+			err = scrapers.ScrapeMazevo(*outDir)
 		case *mapFlag:
-			scrapers.ScrapeMapLocations(*outDir)
+			err = scrapers.ScrapeMapLocations(*outDir)
 		case *academicCalendars:
-			scrapers.ScrapeAcademicCalendars(*outDir)
+			err = scrapers.ScrapeAcademicCalendars(*outDir)
 		case *degrees:
-			scrapers.ScrapeDegrees(*outDir)
+			err = scrapers.ScrapeDegrees(*outDir)
 		case *budgets:
-			scrapers.ScrapeBudgets(*outDir)
+			err = scrapers.ScrapeBudgets(*outDir)
 		default:
 			log.Panic("You must specify which type of scraping you would like to perform with one of the scraping flags!")
 		}
+		if err != nil {
+			log.Panicf("Scraping failed: %v", err)
+		}
 	case *parse:
 		switch {
 		case *cometCalendar:
@@ -186,6 +190,5 @@ func main() {
 		}
 	default:
 		flag.PrintDefaults()
-		return
 	}
 }

parser/cometCalendarParser.go

@@ -220,7 +220,9 @@ func getLocationAbbreviations(inDir string) (map[string]string, []string, error)
 	if err != nil {
 		if os.IsNotExist(err) {
 			// Force scrape the locations if it doesn't exist. Get the map file again
-			scrapers.ScrapeMapLocations(inDir)
+			if err := scrapers.ScrapeMapLocations(inDir); err != nil {
+				return nil, nil, err
+			}
 			ParseMapLocations(inDir, inDir)
 
 			// If it fails to get the locations again, it's not because location is unscraped

scrapers/academicCalendars.go

@@ -25,9 +25,10 @@ type AcademicCalendar struct {
 	Time  string
 }
 
-func ScrapeAcademicCalendars(outDir string) {
+func ScrapeAcademicCalendars(outDir string) error {
 	// Start chromedp
 	chromedpCtx, cancel := utils.InitChromeDp()
+	defer cancel()
 
 	// Get sub folder from output folder
 	outSubDir := filepath.Join(outDir, "academicCalendars")
@@ -36,15 +37,15 @@ func ScrapeAcademicCalendars(outDir string) {
 	os.RemoveAll(outSubDir)
 	err := os.MkdirAll(outSubDir, 0777)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error creating output folder: %w", err)
 	}
 
 	// Go to listings page
 	_, err = chromedp.RunResponse(chromedpCtx,
 		chromedp.Navigate(`https://www.utdallas.edu/academics/calendar/`),
 	)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error navigating to \"https://www.utdallas.edu/academics/calendar/\": %w", err)
 	}
 
 	// Selector for the scraping the calendar nodes
@@ -59,14 +60,14 @@ func ScrapeAcademicCalendars(outDir string) {
 		chromedp.TextContent("h2.wp-block-heading", &academicCalendars[0].Title, chromedp.ByQuery),
 	)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error extracting calendar heading: %w", err)
 	}
 	var currentNode []*cdp.Node
 	err = chromedp.Run(chromedpCtx,
 		chromedp.Nodes(currentSel, &currentNode, chromedp.ByQuery),
 	)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error retrieving the full calendar link: %w", err)
 	}
 	for i := 0; i < len(currentNode[0].Attributes); i += 2 {
 		if currentNode[0].Attributes[i] == "href" {
@@ -80,9 +81,12 @@ func ScrapeAcademicCalendars(outDir string) {
 		chromedp.Nodes(futureSel, &futureNodes, chromedp.BySearch),
 	)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error retrieving future term links: %w", err)
+	}
+	links, err := utils.ExtractTextAndHref(futureNodes, chromedpCtx)
+	if err != nil {
+		return err
 	}
-	links := utils.ExtractTextAndHref(futureNodes, chromedpCtx)
 	for _, link := range links {
 		academicCalendars = append(academicCalendars, AcademicCalendar{
 			Title: link.Text,
@@ -97,9 +101,12 @@ func ScrapeAcademicCalendars(outDir string) {
 		chromedp.Nodes(pastSel, &pastNodes, chromedp.BySearch),
 	)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error retrieving past term links: %w", err)
+	}
+	links, err = utils.ExtractTextAndHref(pastNodes, chromedpCtx)
+	if err != nil {
+		return err
 	}
-	links = utils.ExtractTextAndHref(pastNodes, chromedpCtx)
 	for _, link := range links {
 		academicCalendars = append(academicCalendars, AcademicCalendar{
 			Title: link.Text,
@@ -113,41 +120,45 @@ func ScrapeAcademicCalendars(outDir string) {
 
 	// Download all PDFs
 	for _, academicCalendar := range academicCalendars {
-		downloadPdfFromBox(
+		if err := downloadPdfFromBox(
 			academicCalendar.Href,
 			academicCalendar.Time+"-"+academicCalendar.Title,
 			outSubDir,
-		)
+		); err != nil {
+			return err
+		}
 	}
+	return nil
 }
 
-func downloadPdfFromBox(href string, filename string, outDir string) {
+func downloadPdfFromBox(href string, filename string, outDir string) error {
 	// Create blank file
 	out, err := os.Create(filepath.Join(outDir, fmt.Sprintf("%s.pdf", filename)))
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error creating blank file: %w", err)
 	}
 	defer out.Close()
 
 	// Pull ID from link
 	parsedLink, err := url.Parse(href)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error parsing calendar link: %w", err)
 	}
 	fileId := path.Base(parsedLink.Path)
 
 	// Use box download link with ID
 	resp, err := http.Get(fmt.Sprintf("https://utdallas.box.com/shared/static/%s.pdf", fileId))
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error downloading from box link: %w", err)
 	}
 	defer resp.Body.Close()
 
 	// Output response to blank file
 	_, err = io.Copy(out, resp.Body)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error outputing to blank file: %w", err)
 	}
 
 	log.Printf("Scraped academic calendar %s!", filename)
+	return nil
 }

scrapers/astra.go

@@ -20,14 +20,15 @@ import (
 var MAX_EVENTS_PER_DAY = 5000
 
 // ScrapeAstra iterates day-by-day through Astra events and persists the raw JSON output.
-func ScrapeAstra(outDir string) {
+func ScrapeAstra(outDir string) error {
 	// Start chromedp
 	chromedpCtx, cancel := utils.InitChromeDp()
+	defer cancel()
 
 	// Make output folder
 	err := os.MkdirAll(outDir, 0777)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error creating output folder: %w", err)
 	}
 
 	days := "{"       // String JSON for storing results by day
@@ -42,7 +43,10 @@ func ScrapeAstra(outDir string) {
 	cli := &http.Client{Transport: tr}
 
 	// Get cookies for auth
-	astraHeaders := utils.RefreshAstraToken(chromedpCtx)
+	astraHeaders, err := utils.RefreshAstraToken(chromedpCtx)
+	if err != nil {
+		return fmt.Errorf("error getting cookies for auth: %w", err)
+	}
 	time.Sleep(500 * time.Millisecond)
 	cancel() // Don't need chromedp anymore
 
@@ -60,27 +64,28 @@ func ScrapeAstra(outDir string) {
 		url := fmt.Sprintf("https://www.aaiscloud.com/UTXDallas/~api/calendar/CalendarWeekGrid?_dc=%d&action=GET&start=0&limit=%d&isForWeekView=false&fields=ActivityId,ActivityPk,ActivityName,ParentActivityId,ParentActivityName,MeetingType,Description,StartDate,EndDate,DayOfWeek,StartMinute,EndMinute,ActivityTypeCode,ResourceId,CampusName,BuildingCode,RoomNumber,RoomName,LocationName,InstitutionId,SectionId,SectionPk,IsExam,IsCrosslist,IsAllDay,IsPrivate,EventId,EventPk,CurrentState,NotAllowedUsageMask,UsageColor,UsageColorIsPrimary,EventTypeColor,MaxAttendance,ActualAttendance,Capacity&filter=(StartDate%%3C%%3D%%22%sT23%%3A00%%3A00%%22)%%26%%26(EndDate%%3E%%3D%%22%sT00%%3A00%%3A00%%22)&page=1&sortOrder=%%2BStartDate,%%2BStartMinute", time.Now().UnixMilli(), MAX_EVENTS_PER_DAY, formattedDate, formattedDate)
 		req, err := http.NewRequest("GET", url, nil)
 		if err != nil {
-			panic(err)
+			return fmt.Errorf("error creating http request: %w", err)
 		}
 		req.Header = astraHeaders
 		res, err := cli.Do(req)
 		if err != nil {
-			panic(err)
+			return fmt.Errorf("error sending request: %w", err)
 		}
 		if res.StatusCode != 200 {
-			log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status)
+			res.Body.Close()
+			return fmt.Errorf("error: status was: %s\nIf the status is 404, you've likely been IP ratelimited", res.Status)
 		}
 		body, err := io.ReadAll(res.Body)
+		res.Body.Close()
 		if err != nil {
-			panic(err)
+			return fmt.Errorf("error reading http response body: %w", err)
 		}
-		res.Body.Close()
 		stringBody := string(body)
 
 		// Check for no events
 		numEvents := fastjson.GetInt(body, "totalRecords")
 		if numEvents >= MAX_EVENTS_PER_DAY {
-			log.Panic("ERROR: Max events per day exceeded!")
+			return fmt.Errorf("error: Max events per day exceeded")
 		}
 
 		// Add to record
@@ -98,11 +103,12 @@ func ScrapeAstra(outDir string) {
 	days = fmt.Sprintf("%s}", days)
 	fptr, err := os.Create(fmt.Sprintf("%s/astraScraped.json", outDir))
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error creating output file: %w", err)
 	}
+	defer fptr.Close()
 	_, err = fptr.Write([]byte(days))
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("error writing event data to output file: %w", err)
 	}
-	fptr.Close()
+	return nil
 }