public class HelloScraper : WebScraper{ /// <summary> /// Override this method to initialize your web scraper. /// Important tasks will be to request at least one start URL and set allowed/banned domain or URL patterns. /// </summary> public override voidInit() {License.LicenseKey = "LicenseKey"; // Write License Key this.LoggingLevel = WebScraper.LogLevel.All; // Log all events this.Request("https://blog.scrapinghub.com", Parse); // Initialize a web request to the given URL } /// <summary> /// Override this method to create the default Response handler for your web scraper. /// If you have multiple page types, you can add additional similar methods. /// </summary> /// <param name="response">The HTTP Response object to parse</param> public override voidParse(Response response) { // Set working directory for the project this.WorkingDirectory = AppSetting.GetAppRoot() + @"\HelloScraperSample\Output\"; // Loop on all links foreach (var titleLink in response.Css("h2.entry-title a")) { // Read link text string title = titleLink.TextContentClean; // Save result to fileScrape(new ScrapedData() { { "Title", title } }, "HelloScraper.json"); } // Loop on all links for pagination if (response.CssExists("div.prev-post > a[href]")) { // Get next page URL var nextPage = response.Css("div.prev-post > a[href]")[0].Attributes["href"]; // Scrape next URL this.Request(nextPage, Parse); } }}
public class HelloScraper : WebScraper
{
/// <summary>
/// Override this method to initialize your web scraper.
/// Important tasks will be to request at least one start URL and set allowed/banned domain or URL patterns.
/// </summary>
public override void Init()
{
License.LicenseKey = "LicenseKey"; // Write License Key
this.LoggingLevel = WebScraper.LogLevel.All; // Log all events
this.Request("https://blog.scrapinghub.com", Parse); // Initialize a web request to the given URL
}
/// <summary>
/// Override this method to create the default Response handler for your web scraper.
/// If you have multiple page types, you can add additional similar methods.
/// </summary>
/// <param name="response">The HTTP Response object to parse</param>
public override void Parse(Response response)
{
// Set working directory for the project
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\HelloScraperSample\Output\";
// Loop on all links
foreach (var titleLink in response.Css("h2.entry-title a"))
{
// Read link text
string title = titleLink.TextContentClean;
// Save result to file
Scrape(new ScrapedData() { { "Title", title } }, "HelloScraper.json");
}
// Loop on all links for pagination
if (response.CssExists("div.prev-post > a[href]"))
{
// Get next page URL
var nextPage = response.Css("div.prev-post > a[href]")[0].Attributes["href"];
// Scrape next URL
this.Request(nextPage, Parse);
}
}
}
Public Class HelloScraperInheritsWebScraper ''' <summary> ''' Override this method to initialize your web scraper. ''' Important tasks will be to request at least one start URL and set allowed/banned domain or URL patterns. ''' </summary> PublicOverrides Sub Init()License.LicenseKey = "LicenseKey" ' Write License KeyMe.LoggingLevel = WebScraper.LogLevel.All' Log all eventsMe.Request("https://blog.scrapinghub.com", AddressOfParse) ' Initialize a web request to the given URL End Sub ''' <summary> ''' Override this method to create the default Response handler for your web scraper. ''' If you have multiple page types, you can add additional similar methods. ''' </summary> ''' <param name="response">The HTTP Response object to parse</param> PublicOverrides Sub Parse(ByValresponseAsResponse) ' Set working directory for the projectMe.WorkingDirectory = AppSetting.GetAppRoot() & "\HelloScraperSample\Output\" ' Loop on all links For Each titleLink Inresponse.Css("h2.entry-title a") ' Read link text Dim title AsString = titleLink.TextContentClean ' Save result to fileScrape(New ScrapedData() From { { "Title", title } }, "HelloScraper.json") Next titleLink ' Loop on all links for pagination If response.CssExists("div.prev-post > a[href]") Then ' Get next page URL Dim nextPage = response.Css("div.prev-post > a[href]")(0).Attributes("href") ' Scrape next URLMe.Request(nextPage, AddressOfParse) End If End SubEnd Class
Public Class HelloScraper
Inherits WebScraper
''' <summary>
''' Override this method to initialize your web scraper.
''' Important tasks will be to request at least one start URL and set allowed/banned domain or URL patterns.
''' </summary>
Public Overrides Sub Init()
License.LicenseKey = "LicenseKey" ' Write License Key
Me.LoggingLevel = WebScraper.LogLevel.All ' Log all events
Me.Request("https://blog.scrapinghub.com", AddressOf Parse) ' Initialize a web request to the given URL
End Sub
''' <summary>
''' Override this method to create the default Response handler for your web scraper.
''' If you have multiple page types, you can add additional similar methods.
''' </summary>
''' <param name="response">The HTTP Response object to parse</param>
Public Overrides Sub Parse(ByVal response As Response)
' Set working directory for the project
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\HelloScraperSample\Output\"
' Loop on all links
For Each titleLink In response.Css("h2.entry-title a")
' Read link text
Dim title As String = titleLink.TextContentClean
' Save result to file
Scrape(New ScrapedData() From {
{ "Title", title }
},
"HelloScraper.json")
Next titleLink
' Loop on all links for pagination
If response.CssExists("div.prev-post > a[href]") Then
' Get next page URL
Dim nextPage = response.Css("div.prev-post > a[href]")(0).Attributes("href")
' Scrape next URL
Me.Request(nextPage, AddressOf Parse)
End If
End Sub
End Class
スクレイピングを開始するために、このコードスニペットをMainに追加します
static voidMain(string[] args){ // Create Object From Hello Scrape class HelloScraperSample.HelloScraper scrape = new HelloScraperSample.HelloScraper(); // Start Scraping scrape.Start();}
static void Main(string[] args)
{
// Create Object From Hello Scrape class
HelloScraperSample.HelloScraper scrape = new HelloScraperSample.HelloScraper();
// Start Scraping
scrape.Start();
}
Shared Sub Main(ByVal args() AsString) ' Create Object From Hello Scrape class Dim scrape As New HelloScraperSample.HelloScraper() ' Start Scraping scrape.Start()End Sub
Shared Sub Main(ByVal args() As String)
' Create Object From Hello Scrape class
Dim scrape As New HelloScraperSample.HelloScraper()
' Start Scraping
scrape.Start()
End Sub
namespace IronWebScraperEngine{ public class NewsScraper : IronWebScraper.WebScraper { public override voidInit() { throw new NotImplementedException(); } public override voidParse(Response response) { throw new NotImplementedException(); } }}
namespace IronWebScraperEngine
{
public class NewsScraper : IronWebScraper.WebScraper
{
public override void Init()
{
throw new NotImplementedException();
}
public override void Parse(Response response)
{
throw new NotImplementedException();
}
}
}
NamespaceIronWebScraperEngine Public Class NewsScraperInheritsIronWebScraper.WebScraper PublicOverrides Sub Init()Throw New NotImplementedException() End Sub PublicOverrides Sub Parse(ByValresponseAsResponse)Throw New NotImplementedException() End Sub End ClassEndNamespace
Namespace IronWebScraperEngine
Public Class NewsScraper
Inherits IronWebScraper.WebScraper
Public Overrides Sub Init()
Throw New NotImplementedException()
End Sub
Public Overrides Sub Parse(ByVal response As Response)
Throw New NotImplementedException()
End Sub
End Class
End Namespace
public class MovieScraper : WebScraper{ public override voidInit() {License.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; this.Request("www.website.com", Parse); } public override voidParse(Response response) { foreach (var div in response.Css("#movie-featured > div")) { if (div.GetAttribute("class") != "clearfix") { var movieId = div.GetAttribute("data-movie-id"); var link = div.Css("a")[0]; var movieTitle = link.TextContentClean;Scrape(new ScrapedData() { { "MovieId", movieId }, { "MovieTitle", movieTitle } }, "Movie.Jsonl"); } } }}
public class MovieScraper : WebScraper
{
public override void Init()
{
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
this.Request("www.website.com", Parse);
}
public override void Parse(Response response)
{
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.GetAttribute("class") != "clearfix")
{
var movieId = div.GetAttribute("data-movie-id");
var link = div.Css("a")[0];
var movieTitle = link.TextContentClean;
Scrape(new ScrapedData() { { "MovieId", movieId }, { "MovieTitle", movieTitle } }, "Movie.Jsonl");
}
}
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init()License.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"Me.Request("www.website.com", AddressOfParse) End Sub PublicOverrides Sub Parse(ByValresponseAsResponse) For Each div Inresponse.Css("#movie-featured > div") If div.GetAttribute("class") <> "clearfix" Then Dim movieId = div.GetAttribute("data-movie-id") Dim link = div.Css("a")(0) Dim movieTitle = link.TextContentCleanScrape(New ScrapedData() From { { "MovieId", movieId }, { "MovieTitle", movieTitle } }, "Movie.Jsonl") End If Next div End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
Me.Request("www.website.com", AddressOf Parse)
End Sub
Public Overrides Sub Parse(ByVal response As Response)
For Each div In response.Css("#movie-featured > div")
If div.GetAttribute("class") <> "clearfix" Then
Dim movieId = div.GetAttribute("data-movie-id")
Dim link = div.Css("a")(0)
Dim movieTitle = link.TextContentClean
Scrape(New ScrapedData() From {
{ "MovieId", movieId },
{ "MovieTitle", movieTitle }
},
"Movie.Jsonl")
End If
Next div
End Sub
End Class
構造化ムービークラス
フォーマットされたデータを保持するために、ムービークラスを実装しましょう：
public class Movie{ public intId { get; set; } public stringTitle { get; set; } public stringURL { get; set; }}
public class Movie
{
public int Id { get; set; }
public string Title { get; set; }
public string URL { get; set; }
}
Public Class Movie Public Property IdAsInteger Public Property TitleAsString Public Property URLAsStringEnd Class
Public Class Movie
Public Property Id As Integer
Public Property Title As String
Public Property URL As String
End Class
今、コードを更新してMovieクラスを使用します:
public class MovieScraper : WebScraper{ public override voidInit() {License.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; this.Request("https://website.com/", Parse); } public override voidParse(Response response) { foreach (var div in response.Css("#movie-featured > div")) { if (div.GetAttribute("class") != "clearfix") { var movie = new Movie {Id = Convert.ToInt32(div.GetAttribute("data-movie-id")),Title = div.Css("a")[0].TextContentClean,URL = div.Css("a")[0].Attributes["href"] };Scrape(movie, "Movie.Jsonl"); } } }}
public class MovieScraper : WebScraper
{
public override void Init()
{
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
this.Request("https://website.com/", Parse);
}
public override void Parse(Response response)
{
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.GetAttribute("class") != "clearfix")
{
var movie = new Movie
{
Id = Convert.ToInt32(div.GetAttribute("data-movie-id")),
Title = div.Css("a")[0].TextContentClean,
URL = div.Css("a")[0].Attributes["href"]
};
Scrape(movie, "Movie.Jsonl");
}
}
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init()License.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"Me.Request("https://website.com/", AddressOfParse) End Sub PublicOverrides Sub Parse(ByValresponseAsResponse) For Each div Inresponse.Css("#movie-featured > div") If div.GetAttribute("class") <> "clearfix" Then Dim movie As New MovieWith { .Id = Convert.ToInt32(div.GetAttribute("data-movie-id")), .Title = div.Css("a")(0).TextContentClean, .URL = div.Css("a")(0).Attributes("href") }Scrape(movie, "Movie.Jsonl") End If Next div End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
Me.Request("https://website.com/", AddressOf Parse)
End Sub
Public Overrides Sub Parse(ByVal response As Response)
For Each div In response.Css("#movie-featured > div")
If div.GetAttribute("class") <> "clearfix" Then
Dim movie As New Movie With {
.Id = Convert.ToInt32(div.GetAttribute("data-movie-id")),
.Title = div.Css("a")(0).TextContentClean,
.URL = div.Css("a")(0).Attributes("href")
}
Scrape(movie, "Movie.Jsonl")
End If
Next div
End Sub
End Class
詳細ページのスクレイピング
より詳細な情報を得るために、Movieクラスを拡張しましょう:
public class Movie{ public intId { get; set; } public stringTitle { get; set; } public stringURL { get; set; } public stringDescription { get; set; } public List<string> Genre { get; set; } public List<string> Actor { get; set; }}
public class Movie
{
public int Id { get; set; }
public string Title { get; set; }
public string URL { get; set; }
public string Description { get; set; }
public List<string> Genre { get; set; }
public List<string> Actor { get; set; }
}
Public Class Movie Public Property IdAsInteger Public Property TitleAsString Public Property URLAsString Public Property DescriptionAsString Public Property GenreAsList(OfString) Public Property ActorAsList(OfString)End Class
Public Class Movie
Public Property Id As Integer
Public Property Title As String
Public Property URL As String
Public Property Description As String
Public Property Genre As List(Of String)
Public Property Actor As List(Of String)
End Class
public class MovieScraper : WebScraper{ public override voidInit() {License.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; this.Request("https://domain/", Parse); } public override voidParse(Response response) { foreach (var div in response.Css("#movie-featured > div")) { if (div.GetAttribute("class") != "clearfix") { var movie = new Movie {Id = Convert.ToInt32(div.GetAttribute("data-movie-id")),Title = div.Css("a")[0].TextContentClean,URL = div.Css("a")[0].Attributes["href"] }; this.Request(movie.URL, ParseDetails, new MetaData() { { "movie", movie } }); } } } public voidParseDetails(Response response) { var movie = response.MetaData.Get<Movie>("movie"); var div = response.Css("div.mvic-desc")[0]; movie.Description = div.Css("div.desc")[0].TextContentClean; movie.Genre = div.Css("div > p > a").Select(element => element.TextContentClean).ToList(); movie.Actor = div.Css("div > p:nth-child(2) > a").Select(element => element.TextContentClean).ToList();Scrape(movie, "Movie.Jsonl"); }}
public class MovieScraper : WebScraper
{
public override void Init()
{
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
this.Request("https://domain/", Parse);
}
public override void Parse(Response response)
{
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.GetAttribute("class") != "clearfix")
{
var movie = new Movie
{
Id = Convert.ToInt32(div.GetAttribute("data-movie-id")),
Title = div.Css("a")[0].TextContentClean,
URL = div.Css("a")[0].Attributes["href"]
};
this.Request(movie.URL, ParseDetails, new MetaData() { { "movie", movie } });
}
}
}
public void ParseDetails(Response response)
{
var movie = response.MetaData.Get<Movie>("movie");
var div = response.Css("div.mvic-desc")[0];
movie.Description = div.Css("div.desc")[0].TextContentClean;
movie.Genre = div.Css("div > p > a").Select(element => element.TextContentClean).ToList();
movie.Actor = div.Css("div > p:nth-child(2) > a").Select(element => element.TextContentClean).ToList();
Scrape(movie, "Movie.Jsonl");
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init()License.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"Me.Request("https://domain/", AddressOfParse) End Sub PublicOverrides Sub Parse(ByValresponseAsResponse) For Each div Inresponse.Css("#movie-featured > div") If div.GetAttribute("class") <> "clearfix" Then Dim movie As New MovieWith { .Id = Convert.ToInt32(div.GetAttribute("data-movie-id")), .Title = div.Css("a")(0).TextContentClean, .URL = div.Css("a")(0).Attributes("href") }Me.Request(movie.URL, AddressOfParseDetails, New MetaData() From { { "movie", movie } }) End If Next div End Sub Public Sub ParseDetails(ByValresponseAsResponse) Dim movie = response.MetaData.Get(OfMovie)("movie") Dim div = response.Css("div.mvic-desc")(0) movie.Description = div.Css("div.desc")(0).TextContentClean movie.Genre = div.Css("div > p > a").Select(Function(element) element.TextContentClean).ToList() movie.Actor = div.Css("div > p:nth-child(2) > a").Select(Function(element) element.TextContentClean).ToList()Scrape(movie, "Movie.Jsonl") End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
Me.Request("https://domain/", AddressOf Parse)
End Sub
Public Overrides Sub Parse(ByVal response As Response)
For Each div In response.Css("#movie-featured > div")
If div.GetAttribute("class") <> "clearfix" Then
Dim movie As New Movie With {
.Id = Convert.ToInt32(div.GetAttribute("data-movie-id")),
.Title = div.Css("a")(0).TextContentClean,
.URL = div.Css("a")(0).Attributes("href")
}
Me.Request(movie.URL, AddressOf ParseDetails, New MetaData() From {
{ "movie", movie }
})
End If
Next div
End Sub
Public Sub ParseDetails(ByVal response As Response)
Dim movie = response.MetaData.Get(Of Movie)("movie")
Dim div = response.Css("div.mvic-desc")(0)
movie.Description = div.Css("div.desc")(0).TextContentClean
movie.Genre = div.Css("div > p > a").Select(Function(element) element.TextContentClean).ToList()
movie.Actor = div.Css("div > p:nth-child(2) > a").Select(Function(element) element.TextContentClean).ToList()
Scrape(movie, "Movie.Jsonl")
End Sub
End Class
PublicOverrides Sub Init()License.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\ShoppingSiteSample\Output\"EnableWebCache()Me.Request("http://www.WebSite.com", Parse)End Sub
Public Overrides Sub Init()
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\ShoppingSiteSample\Output\"
EnableWebCache()
Me.Request("http://www.WebSite.com", Parse)
End Sub
How can developers extend the functionality of IronWebScraper for specific data extraction needs?
Developers can inherit from the IronWebScraper.WebScraper class to extend functionality and implement custom data extraction logic by overriding methods like Init() and Parse(Response response).
