-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathProgram.cs
More file actions
64 lines (59 loc) · 2.19 KB
/
Copy pathProgram.cs
File metadata and controls
64 lines (59 loc) · 2.19 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
using System;
using System.Collections.Generic;
using HtmlAgilityPack;
using ScrapySharp.Extensions;
using ScrapySharp.Network;
namespace HTML_SCRAPE
{
class Program
{
static void Main(string[] args)
{
var mainPageLinks = GetMainPageLinks("https://newyork.craigslist.org/d/computer-gigs/search/cpg");
var lstGigs = GetPageDetails(mainPageLinks);
}
static HtmlNode GetHtml(string url)
{
WebPage webpage = _browser.NavigateToPage(new Uri(url));
return webpage.Html;
}
static List<string> GetMainPageLinks(string url)
{
var homePageLinks = new List<string>();
var html = GetHtml(url);
var links = html.CssSelect("a");
foreach (var link in links)
{
if (link.Attributes["href"].Value.Contains(".html"))
{
homePageLinks.Add(link.Attributes["href"].Value);
}
}
return homePageLinks;
}
public class PageDetails
{//Define class for scrape info
public string title { get; set; }
public string description { get; set; }
public string url { get; set; }
}
static List<PageDetails> GetPageDetails(List<string> urls)
{//Define method for getting info from html
var lstPageDetails = new List<PageDetails>();
foreach (var url in urls)
{
var htmlNode = GetHtml(url);
var pageDetails = new PageDetails();
pageDetails.title = htmlNode.OwnerDocument.DocumentNode
.SelectSingleNode("//html/head/title").InnerText;
var description = htmlNode.OwnerDocument.DocumentNode
.SelectSingleNode("//html/body/section/section/section/section").InnerText;
pageDetails.description = description
.Replace("\n \n QR Code Link to This Post\n \n \n", "");
pageDetails.url = url;
lstPageDetails.Add(pageDetails);
}
return lstPageDetails;
}
}
}