1package content
 2
 3import (
 4	"encoding/json"
 5	"fmt"
 6	"io"
 7	"net/http"
 8	"os"
 9	"path/filepath"
10	"time"
11)
12
13func NowHex() string {
14	return fmt.Sprintf("%x", time.Now().Unix())
15}
16
17func (item *Item) SnapshotDir() string {
18	return filepath.Join(item.Id.Dir(), "raw", NowHex())
19}
20
21func (item *Item) Fetch() error {
22
23	req, err := http.NewRequest("GET", item.TargetURL.String(), nil)
24	if err != nil {
25		return err
26	}
27	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0")
28
29	client := &http.Client{
30		Timeout: 10 * time.Second,
31	}
32	resp, err := client.Do(req)
33	if err != nil {
34		return err
35	}
36	defer resp.Body.Close()
37
38	snapshotDir := item.SnapshotDir()
39	err = os.MkdirAll(snapshotDir, _dirMode)
40	if err != nil {
41		return err
42	}
43
44	headersPath := filepath.Join(snapshotDir, "headers.json")
45	headers, err := os.Create(headersPath)
46	if err != nil {
47		return err
48	}
49	err = json.NewEncoder(headers).Encode(resp.Header)
50	if err != nil {
51		return err
52	}
53
54	statusPath := filepath.Join(snapshotDir, "status.txt")
55	status, err := os.Create(statusPath)
56	if err != nil {
57		return err
58	}
59	_, err = status.WriteString(resp.Status)
60	if err != nil {
61		return err
62	}
63
64	if resp.StatusCode != http.StatusOK {
65		// skip writing body
66		fmt.Printf("\tstatus: %s\n", resp.Status)
67		return nil
68	}
69
70	bodyPath := filepath.Join(snapshotDir, "body")
71	body, err := os.Create(bodyPath)
72	if err != nil {
73		return err
74	}
75	_, err = io.Copy(body, resp.Body)
76	if err != nil {
77		return err
78	}
79
80	return nil
81}