console

type CqlModel struct {
	Cluster *gocql.ClusterConfig
	Db      *gocql.Session
}

type DomainInfo struct {
	//TLD+1
	Domain string

	//Why did this domain get excluded, or empty if not excluded
	ExcludeReason string

	//When did this domain last get queued to be crawled. Or TimeQueed.IsZero() if not crawled
	TimeQueued time.Time

	//What was the UUID of the crawler that last crawled the domain
	UuidOfQueued gocql.UUID

	//Number of (unique) links found in this domain
	NumberLinksTotal int

	//Number of (unique) links queued to be processed for this domain
	NumberLinksQueued int
}

type LinkInfo struct {
	//URL of the link
	Url string

	//Status of the GET
	Status int

	//Any error reported during the get
	Error string

	//Was this excluded by robots
	RobotsExcluded bool

	//When did this link get crawled
	CrawlTime time.Time
}

type Model interface {

	// Close the data store after you're done with it
	Close()

	// InsertLinks queues a set of URLS to be crawled
	InsertLinks(links []string) []error

	// Find a specific domain
	FindDomain(domain string) (*DomainInfo, error)

	// List domains
	ListDomains(seedDomain string, limit int) ([]DomainInfo, error)

	// Same as ListDomains, but only lists the domains that are currently queued
	ListWorkingDomains(seedDomain string, limit int) ([]DomainInfo, error)

	// List links from the given domain
	ListLinks(domain string, seedUrl string, limit int) ([]LinkInfo, error)

	// For a given linkUrl, return the entire crawl history
	ListLinkHistorical(linkUrl string, seedIndex int, limit int) ([]LinkInfo, int, error)

	// Find a link
	FindLink(links string) (*LinkInfo, error)
}

type Route struct {
	Path       string
	Controller func(w http.ResponseWriter, req *http.Request)
}