fix: set proper User-Agent and add retry delay

colly's default User-Agent identifies requests as coming from a scraping framework,
which causes CDNs like Cloudflare to silently drop connections from CI runners
This commit sets a descriptive bot-style User-Agent instead. It also adds a 2s delay
before retrying on status 0 and 503, matching the existing retry logic

Signed-off-by: vprashar2929 <vibhu.sharma2929@gmail.com>

Author: vprashar2929

pkg/mdformatter/linktransformer/link.go

@@ -218,7 +218,7 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []
 	linktransformerMetrics := newLinktransformerMetrics(reg)
 	transport := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
-		ForceAttemptHTTP2:     true,
+		ForceAttemptHTTP2:     false,
 		MaxIdleConns:          100,
 		IdleConnTimeout:       90 * time.Second,
 		TLSHandshakeTimeout:   10 * time.Second,
@@ -238,7 +238,7 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []
 		validateConfig: config,
 		localLinks:     map[string]*[]string{},
 		remoteLinks:    map[string]error{},
-		c:              colly.NewCollector(colly.Async(), colly.StdlibContext(ctx)),
+		c:              colly.NewCollector(colly.Async(), colly.StdlibContext(ctx), colly.UserAgent("Mozilla/5.0 (compatible; mdox/link-checker; +https://github.com/bwplotka/mdox)")),
 		storage:        nil,
 		destFutures:    map[futureKey]*futureResult{},
 		l:              linktransformerMetrics,
@@ -262,6 +262,7 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []
 	// on API (only search is below 100).
 	if config.Timeout != "" {
 		v.c.SetRequestTimeout(config.timeout)
+		transport.ResponseHeaderTimeout = config.timeout
 	}
 
 	if v.validateConfig.Cache.IsSet() && storage != nil {
@@ -334,7 +335,11 @@ func NewValidator(ctx context.Context, logger log.Logger, linksValidateConfig []
 				break
 			}
 			response.Ctx.Put(numberOfRetriesKey, strconv.Itoa(retries+1))
-
+			select {
+			case <-time.After(2 * time.Second):
+			case <-v.c.Context.Done():
+				return
+			}
 			if retryErr := response.Request.Retry(); retryErr != nil {
 				v.remoteLinks[response.Ctx.Get(originalURLKey)] = fmt.Errorf("remote link retry %v: %w", response.Ctx.Get(originalURLKey), err)
 				break