gtfsdb/main.go

// gtfsdb is a command line tool to roundtrip GTFS data to MySQL..
// It's especially powerful when used with Dolt.
package main

import (
	"archive/zip"
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
	"encoding/csv"
)

type GTFS map[string][]string

func readFromURL(url string) GTFS {
	// Get GTFS zip file from URL
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer resp.Body.Close()

	// Read all into a byte slice
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Read zip file
	return readFromZip(body)
}

func readFromFile(filename string) GTFS {
	// Read GTFS zip file from filename
	file, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer file.Close()

	// Read all into a byte slice
	fbytes, err := io.ReadAll(file)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Read zip file
	return readFromZip(fbytes)
}

func readFromZip(file []byte) GTFS {
	// Open zip file
	r, err := zip.NewReader(bytes.NewReader(file), int64(len(file)))
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Read all files in zip
	gtfs := GTFS{}
	for _, f := range r.File {
		// Open file
		rc, err := f.Open()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		defer rc.Close()

		// Read CSV file. Remove .txt extension.
		gtfs[f.Name[:len(f.Name)-4]] = readFromCSV(rc)
	}

	return gtfs
}

func readFromCSV(file io.Reader) []string {
	// Read CSV file
	r := csv.NewReader(file)
	rows, err := r.ReadAll()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Convert rows to []string
	var result []string
	for _, row := range rows {
		result = append(result, strings.Join(row, ","))
	}

	return result
}

func writeToDir(gtfs GTFS, ext string) {
	for table, rows := range gtfs {
		// Open file
		file, err := os.Create(table + "." + ext)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		defer file.Close()

		// Write rows
		for _, row := range rows {
			_, err := io.WriteString(file, row+"\n")
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
		}
	}
}

func main() {
	// Read filename from command line and write contents of zip to current directory as CSV files
	var filename string
	if len(os.Args) > 1 {
		filename = os.Args[1]
	} else {
		fmt.Println("Please provide a URL or filename")
		os.Exit(1)
	}

	// Read GTFS zip file from URL or filename
	var gtfs GTFS
	if strings.HasPrefix(filename, "http") {
		gtfs = readFromURL(filename)
	} else {
		gtfs = readFromFile(filename)
	}

	// Write GTFS to current directory
	writeToDir(gtfs, "csv")
}