1package content
  2
  3import (
  4	"encoding/json"
  5	"errors"
  6	"net/url"
  7	"os"
  8	"time"
  9)
 10
 11const (
 12	_dirMode  = 0o750
 13	_fileMode = 0o640
 14)
 15
 16// Item is an indexed
 17type Item struct {
 18	Id        URLID
 19	Title     string
 20	TargetURL *url.URL
 21
 22	FirstSeen    time.Time
 23	LastSeen     time.Time
 24	Fetched      time.Time
 25	Observations map[URLID]*Observation
 26}
 27
 28func NewItem(
 29	url *url.URL,
 30	title string,
 31) (*Item, error) {
 32	id, err := NewURLID(url)
 33	if err != nil {
 34		return nil, err
 35	}
 36	return &Item{
 37		Id:        id,
 38		Title:     title,
 39		TargetURL: url,
 40	}, nil
 41}
 42
 43// Upsert reads an item from disk (if it exists) and updates the data
 44func Upsert(o *Observation) (*Item, error) {
 45	id, err := NewURLID(o.TargetURL)
 46	if err != nil {
 47		return nil, err
 48	}
 49
 50	item, err := id.GetItem()
 51	switch {
 52
 53	case err == nil:
 54	// item loaded, continue
 55
 56	case errors.Is(err, ErrNotFound):
 57		item, err = NewItem(o.TargetURL, o.Title)
 58		if err != nil {
 59			return nil, err
 60		}
 61
 62	default:
 63		return nil, err
 64	}
 65
 66	// update item
 67	now := time.Now()
 68	if item.FirstSeen.IsZero() {
 69		item.FirstSeen = now
 70	}
 71	if item.Title == "" {
 72		item.Title = o.Title
 73	}
 74	if item.Observations == nil {
 75		item.Observations = map[URLID]*Observation{}
 76	}
 77	item.Observations[o.Id] = o
 78	item.LastSeen = now
 79
 80	err = item.WriteMetadata()
 81	if err != nil {
 82		return nil, err
 83	}
 84
 85	return item, nil
 86}
 87
 88func (item *Item) WriteMetadata() error {
 89
 90	dir := item.Id.Dir()
 91	temp := item.Id.Temp()
 92	path := item.Id.Path()
 93
 94	err := os.MkdirAll(dir, _dirMode)
 95	if err != nil {
 96		return err
 97	}
 98
 99	f, err := os.Create(temp)
100	if err != nil {
101		return err
102	}
103	defer func() { _ = f.Close() }()
104
105	err = json.NewEncoder(f).Encode(item)
106	if err != nil {
107		return err
108	}
109
110	return os.Rename(temp, path)
111}
112
113// MarshalJSON allows for stringified urls in marshaled Item type
114// satisfies the json.Marshaler interface
115func (item *Item) MarshalJSON() ([]byte, error) {
116	i := struct {
117		Id        URLID  `json:"id"`
118		Title     string `json:"title"`
119		TargetURL string `json:"target_url"`
120
121		FirstSeen    time.Time              `json:"first_seen,omitempty"`
122		LastSeen     time.Time              `json:"last_seen,omitempty"`
123		Fetched      time.Time              `json:"fetched,omitempty"`
124		Observations map[URLID]*Observation `json:"observations,omitempty"`
125	}{
126		Id:           item.Id,
127		Title:        item.Title,
128		TargetURL:    item.TargetURL.String(),
129		FirstSeen:    item.FirstSeen,
130		LastSeen:     item.LastSeen,
131		Fetched:      item.Fetched,
132		Observations: item.Observations,
133	}
134	return json.Marshal(&i)
135}
136
137// UnmarshalJSON parses string urls from the json bytes to url.URL where needed
138// satisfies the json.Unmarshaler interface
139func (item *Item) UnmarshalJSON(data []byte) error {
140	i := struct {
141		Id        URLID  `json:"id"`
142		Title     string `json:"title"`
143		TargetURL string `json:"target_url"`
144
145		FirstSeen    *time.Time             `json:"first_seen,omitempty"`
146		LastSeen     *time.Time             `json:"last_seen,omitempty"`
147		Fetched      *time.Time             `json:"fetched,omitempty"`
148		Observations map[URLID]*Observation `json:"observations,omitempty"`
149	}{}
150	err := json.Unmarshal(data, &i)
151	if err != nil {
152		return err
153	}
154
155	u, err := url.Parse(i.TargetURL)
156	if err != nil {
157		return err
158	}
159
160	item.Id = i.Id
161	item.Title = i.Title
162	item.TargetURL = u
163
164	if i.FirstSeen != nil {
165		item.FirstSeen = *i.FirstSeen
166	}
167
168	if i.LastSeen != nil {
169		item.LastSeen = *i.LastSeen
170	}
171
172	if i.Fetched != nil {
173		item.Fetched = *i.Fetched
174	}
175
176	item.Observations = i.Observations
177
178	return nil
179}