Skip to content

put accept header before OnRequest #786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 20, 2023
Merged

put accept header before OnRequest #786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
693a39f
put accept header before OnRequest
k4lizen Oct 3, 2023
e08409f
added NoAcceptHeader unit test
k4lizen Oct 17, 2023
a485676
added check for Accept header in Response.Request
k4lizen Oct 17, 2023
e181623
fixed check for accept header, added accept_header to test server
k4lizen Oct 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions colly.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -671,6 +671,10 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct
ID: atomic.AddUint32(&c.requestCount, 1),
}

if req.Header.Get("Accept") == "" {
req.Header.Set("Accept", "*/*")
}

c.handleOnRequest(request)

if request.abort {
Expand All @@ -681,10 +685,6 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
}

if req.Header.Get("Accept") == "" {
req.Header.Set("Accept", "*/*")
}

var hTrace *HTTPTrace
if c.TraceHTTP {
hTrace = &HTTPTrace{}
Expand Down
38 changes: 38 additions & 0 deletions colly_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,11 @@ func newUnstartedTestServer() *httptest.Server {
w.Write([]byte(r.Host))
})

mux.HandleFunc("/accept_header", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
w.Write([]byte(r.Header.Get("Accept")))
})

mux.HandleFunc("/custom_header", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
w.Write([]byte(r.Header.Get("Test")))
Expand Down Expand Up @@ -424,6 +429,39 @@ var newCollectorTests = map[string]func(*testing.T){
},
}

func TestNoAcceptHeader(t *testing.T) {
ts := newTestServer()
defer ts.Close()

var receivedHeader string
// checks if Accept is enabled by default
func() {
c := NewCollector()
c.OnResponse(func(resp *Response) {
receivedHeader = string(resp.Body)
})
c.Visit(ts.URL + "/accept_header")
if receivedHeader != "*/*" {
t.Errorf("default Accept header isn't */*. got: %v", receivedHeader)
}
}()

// checks if Accept can be disabled
func() {
c := NewCollector()
c.OnRequest(func(r *Request) {
r.Headers.Del("Accept")
})
c.OnResponse(func(resp *Response) {
receivedHeader = string(resp.Body)
})
c.Visit(ts.URL + "/accept_header")
if receivedHeader != "" {
t.Errorf("failed to pass request with no Accept header. got: %v", receivedHeader)
}
}()
}

func TestNewCollector(t *testing.T) {
t.Run("Functional Options", func(t *testing.T) {
for name, test := range newCollectorTests {
Expand Down