Merge pull request #68 from TurnerSoftware/fix-max-pages-allowed-bug

Fix max pages allowed bug

Author: Turnerj

src/Directory.build.props src/Directory.Build.propssrc/Directory.build.props renamed to src/Directory.Build.props

@@ -63,13 +63,12 @@ public async Task<CrawlResult> Crawl(Uri siteUri, CrawlSettings settings)
 
 			result.CrawledUris = await crawlRunner.ProcessAsync(async (requestResult, crawlState) =>
 			{
-				var response = requestResult.ResponseMessage;
-				using (var contentStream = await response.Content.ReadAsStreamAsync())
+				using (requestResult.Content)
 				{
-					var headers = new CrawlHeaders(response.Headers, response.Content.Headers);
-					var content = settings.ContentProcessor.Parse(crawlState.Location, headers, contentStream);
-					contentStream.Seek(0, SeekOrigin.Begin);
-					content.RawContent = await new StreamReader(contentStream).ReadToEndAsync();
+					var headers = new CrawlHeaders(requestResult.ResponseHeaders, requestResult.ContentHeaders);
+					var content = settings.ContentProcessor.Parse(crawlState.Location, headers, requestResult.Content);
+					requestResult.Content.Seek(0, SeekOrigin.Begin);
+					content.RawContent = await new StreamReader(requestResult.Content).ReadToEndAsync();
 					crawlRunner.AddResult(crawlState.Location, content);
 				}
 			});

src/InfinityCrawler/Crawler.cs

@@ -50,7 +50,7 @@ private Uri StripFragment(Uri uri)
 			}.Uri;
 		}
 
-		public void AddLink(CrawlLink crawlLink)
+		private void AddLink(CrawlLink crawlLink)
 		{
 			if (crawlLink.Relationship != null && crawlLink.Relationship.Equals("nofollow", StringComparison.InvariantCultureIgnoreCase))
 			{
@@ -109,14 +109,6 @@ public void AddResult(Uri requestUri, CrawledContent content)
 				{
 					Logger?.LogDebug($"Result for {requestUri} has completed successfully with content.");
 
-					if (robotsPageDefinition.CanFollowLinks(Settings.UserAgent))
-					{
-						foreach (var crawlLink in content.Links)
-						{
-							AddLink(crawlLink);
-						}
-					}
-
 					AddResult(new CrawledUri
 					{
 						Location = crawlState.Location,
@@ -125,6 +117,14 @@ public void AddResult(Uri requestUri, CrawledContent content)
 						Requests = crawlState.Requests,
 						Content = content
 					});
+
+					if (robotsPageDefinition.CanFollowLinks(Settings.UserAgent))
+					{
+						foreach (var crawlLink in content.Links)
+						{
+							AddLink(crawlLink);
+						}
+					}
 				}
 			}
 		}
@@ -232,7 +232,7 @@ await Settings.RequestProcessor.ProcessAsync(
 						Location = requestResult.RequestUri
 					});
 
-					if (requestResult.ResponseMessage == null)
+					if (requestResult.Exception != null)
 					{
 						//Retry failed requests
 						Logger?.LogDebug($"An exception occurred while requesting {crawlState.Location}. This URL will be added to the request queue to be attempted again later.");
@@ -245,14 +245,12 @@ await Settings.RequestProcessor.ProcessAsync(
 					}
 					else
 					{
-						var response = requestResult.ResponseMessage;
-
 						var crawlRequest = new CrawlRequest
 						{
 							RequestStart = requestResult.RequestStart,
 							ElapsedTime = requestResult.ElapsedTime,
-							StatusCode = response.StatusCode,
-							IsSuccessfulStatus = response.IsSuccessStatusCode
+							StatusCode = requestResult.StatusCode,
+							IsSuccessfulStatus = (int)requestResult.StatusCode is >= 200 and <= 299
 						};
 						crawlState.Requests.Add(crawlRequest);
 
@@ -264,8 +262,8 @@ await Settings.RequestProcessor.ProcessAsync(
 						};
 						if (redirectStatusCodes.Contains(crawlRequest.StatusCode.Value))
 						{
-							Logger?.LogDebug($"Result for {crawlState.Location} was a redirect ({response.Headers.Location}). This URL will be added to the request queue.");
-							AddRedirect(crawlState.Location, response.Headers.Location);
+							Logger?.LogDebug($"Result for {crawlState.Location} was a redirect ({requestResult.ResponseHeaders.Location}). This URL will be added to the request queue.");
+							AddRedirect(crawlState.Location, requestResult.ResponseHeaders.Location);
 						}
 						else if (crawlRequest.IsSuccessfulStatus)
 						{

src/InfinityCrawler/Internal/CrawlRunner.cs

@@ -39,7 +39,7 @@ public async Task ProcessAsync(HttpClient httpClient, Func<RequestResult, Task>
 			}
 
 			var random = new Random();
-			var activeRequests = new ConcurrentDictionary<Task, RequestContext>(options.MaxNumberOfSimultaneousRequests, options.MaxNumberOfSimultaneousRequests);
+			var activeRequests = new ConcurrentDictionary<Task<RequestResult>, RequestContext>(options.MaxNumberOfSimultaneousRequests, options.MaxNumberOfSimultaneousRequests);
 
 			var currentBackoff = 0;
 			var successesSinceLastThrottle = 0;
@@ -77,7 +77,7 @@ public async Task ProcessAsync(HttpClient httpClient, Func<RequestResult, Task>
 
 						Logger?.LogDebug($"Request #{requestContext.RequestNumber} ({requestUri}) starting with a {requestStartDelay}ms delay.");
 
-						var task = PerformRequestAsync(httpClient, responseAction, requestContext);
+						var task = PerformRequestAsync(httpClient, requestContext);
 
 						activeRequests.TryAdd(task, requestContext);
 						requestCount++;
@@ -91,6 +91,8 @@ public async Task ProcessAsync(HttpClient httpClient, Func<RequestResult, Task>
 
 				await Task.WhenAny(activeRequests.Keys).ConfigureAwait(false);
 
+				cancellationToken.ThrowIfCancellationRequested();
+
 				var completedRequests = activeRequests.Keys.Where(t => t.IsCompleted);
 				foreach (var completedRequest in completedRequests)
 				{
@@ -105,6 +107,8 @@ public async Task ProcessAsync(HttpClient httpClient, Func<RequestResult, Task>
 						ExceptionDispatchInfo.Capture(aggregateException.InnerException).Throw();
 					}
 
+					await responseAction(completedRequest.Result);
+
 					//Manage the throttling based on timeouts and successes
 					var throttlePoint = options.TimeoutBeforeThrottle;
 					if (throttlePoint.TotalMilliseconds > 0 && requestContext.Timer.Elapsed > throttlePoint)
@@ -130,7 +134,7 @@ public async Task ProcessAsync(HttpClient httpClient, Func<RequestResult, Task>
 			Logger?.LogDebug($"Completed processing {requestCount} requests.");
 		}
 
-		private async Task PerformRequestAsync(HttpClient httpClient, Func<RequestResult, Task> responseAction, RequestContext context)
+		private async Task<RequestResult> PerformRequestAsync(HttpClient httpClient, RequestContext context)
 		{
 			if (context.RequestStartDelay > 0)
 			{
@@ -146,7 +150,9 @@ private async Task PerformRequestAsync(HttpClient httpClient, Func<RequestResult
 				var combinedToken = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, timeoutToken).Token;
 				using (var response = await httpClient.GetAsync(context.RequestUri, combinedToken))
 				{
-					await response.Content.LoadIntoBufferAsync();
+					var contentStream = new MemoryStream();
+					await response.Content.CopyToAsync(contentStream);
+					contentStream.Seek(0, SeekOrigin.Begin);
 
 					//We only want to time the request, not the handling of the response
 					context.Timer.Stop();
@@ -155,19 +161,23 @@ private async Task PerformRequestAsync(HttpClient httpClient, Func<RequestResult
 
 					Logger?.LogDebug($"Request #{context.RequestNumber} completed successfully in {context.Timer.ElapsedMilliseconds}ms.");
 
-					await responseAction(new RequestResult
+					return new RequestResult
 					{
 						RequestUri = context.RequestUri,
 						RequestStart = requestStart,
 						RequestStartDelay = context.RequestStartDelay,
-						ResponseMessage = response,
+						StatusCode = response.StatusCode,
+						ResponseHeaders = response.Headers,
+						ContentHeaders = response.Content.Headers,
+						Content = contentStream,
 						ElapsedTime = context.Timer.Elapsed
-					});
+					};
 				}
 			}
 			catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested)
 			{
 				Logger?.LogDebug($"Request #{context.RequestNumber} cancelled.");
+				return null;
 			}
 			catch (Exception ex) when (ex is HttpRequestException || ex is OperationCanceledException)
 			{
@@ -176,14 +186,14 @@ await responseAction(new RequestResult
 				Logger?.LogDebug($"Request #{context.RequestNumber} completed with error in {context.Timer.ElapsedMilliseconds}ms.");
 				Logger?.LogTrace(ex, $"Request #{context.RequestNumber} Exception: {ex.Message}");
 
-				await responseAction(new RequestResult
+				return new RequestResult
 				{
 					RequestUri = context.RequestUri,
 					RequestStart = requestStart,
 					RequestStartDelay = context.RequestStartDelay,
 					ElapsedTime = context.Timer.Elapsed,
 					Exception = ex
-				});
+				};
 			}
 		}
 	}

src/InfinityCrawler/Processing/Requests/DefaultRequestProcessor.cs

@@ -1,6 +1,9 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Net;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Text;
 
 namespace InfinityCrawler.Processing.Requests
@@ -10,7 +13,10 @@ public class RequestResult
 		public Uri RequestUri { get; set; }
 		public DateTime RequestStart { get; set; }
 		public double RequestStartDelay { get; set; }
-		public HttpResponseMessage ResponseMessage { get; set; }
+		public HttpStatusCode? StatusCode { get; set; }
+		public HttpResponseHeaders ResponseHeaders { get; set; }
+		public HttpContentHeaders ContentHeaders { get; set; }
+		public Stream Content { get; set; }
 		public TimeSpan ElapsedTime { get; set; }
 		public Exception Exception { get; set; }
 	}

src/InfinityCrawler/Processing/Requests/RequestResult.cs

@@ -141,9 +141,11 @@ public async Task MaximumRedirectLimitFollowed()
 			Assert.AreEqual(CrawlStatus.MaxRedirects, crawledUri.Status);
 			Assert.AreEqual(3, crawledUri.RedirectChain.Count);
 		}
-		
-		[TestMethod]
-		public async Task MaximumPagesCrawledFollowed()
+
+		[DataRow(2)]
+		[DataRow(4)]
+		[DataTestMethod]
+		public async Task MaximumPagesCrawledFollowed(int maxPages)
 		{
 			var crawler = GetTestSiteCrawler(new SiteContext
 			{
@@ -155,15 +157,11 @@ public async Task MaximumPagesCrawledFollowed()
 				RequestProcessorOptions = GetNoDelayRequestProcessorOptions()
 			};
 
-			settings.MaxNumberOfPagesToCrawl = 4;
+			settings.MaxNumberOfPagesToCrawl = maxPages;
 			var result = await crawler.Crawl(new Uri("http://localhost/"), settings);
-			Assert.AreEqual(4, result.CrawledUris.Count());
-
-			settings.MaxNumberOfPagesToCrawl = 2;
-			result = await crawler.Crawl(new Uri("http://localhost/"), settings);
-			Assert.AreEqual(2, result.CrawledUris.Count());
+			Assert.AreEqual(maxPages, result.CrawledUris.Count());
 		}
-		
+
 		[TestMethod]
 		public async Task AutoRetryOnFailure()
 		{
@@ -185,10 +183,10 @@ public async Task AutoRetryOnFailure()
 				}
 			};
 
-			settings.RequestProcessor.Add(new Uri("http://localhost/delay/300/300ms-delay-1"));
-			settings.RequestProcessor.Add(new Uri("http://localhost/delay/300/300ms-delay-2"));
-			settings.RequestProcessor.Add(new Uri("http://localhost/delay/300/300ms-delay-3"));
-			settings.RequestProcessor.Add(new Uri("http://localhost/delay/300/300ms-delay-4"));
+			settings.RequestProcessor.Add(new Uri("http://localhost/delay/500/500ms-delay-1"));
+			settings.RequestProcessor.Add(new Uri("http://localhost/delay/500/500ms-delay-2"));
+			settings.RequestProcessor.Add(new Uri("http://localhost/delay/500/500ms-delay-3"));
+			settings.RequestProcessor.Add(new Uri("http://localhost/delay/500/500ms-delay-4"));
 
 			var results = await crawler.Crawl(new Uri("http://localhost/"), settings);
 			var delayedCrawls = results.CrawledUris.Where(c => c.Location.PathAndQuery.Contains("delay")).ToArray();