IronWebScraper通過解析HTML元素從網站中提取電影資料，建立型別化物件以進行結構化資料儲存，並使用元資料在頁面間導航，以構建全面的電影資訊資料集。 這個C# Web Scraper程式庫簡化了將非結構化的網站內容轉換為有組織的、可分析的資料的過程。
快速入門：在C#中爬取電影
通過NuGet Package Manager安裝IronWebScraper
建立一個繼承自WebScraper的類
重寫Init()以設置授權並請求目標URL
重寫Parse()以使用CSS選擇器提取電影資料
使用Scrape()方法以JSON格式保存資料
1Install IronWebScraper with NuGet Package Manager
PM > Install-Package IronWebScraper
Install-Package IronWebScraper
2複製並運行這段程式碼片段。
using IronWebScraper;using System;public class QuickstartMovieScraper : WebScraper{ public override voidInit() { // Set your license keyLicense.LicenseKey = "YOUR-LICENSE-KEY"; // Configure scraper settings this.LoggingLevel = LogLevel.All; this.WorkingDirectory = @"C:\MovieData\Output\"; // Start scraping from the homepage this.Request("https://example-movie-site.com", Parse); } public override voidParse(Response response) { // Extract movie titles using CSS selectors foreach (var movieDiv in response.Css(".movie-item")) { var title = movieDiv.Css("h2")[0].TextContentClean; var url = movieDiv.Css("a")[0].Attributes["href"]; // Save the scraped dataScrape(new { Title = title, Url = url }, "movies.json"); } }}// Run the scrapervar scraper = new QuickstartMovieScraper();scraper.Start();
using IronWebScraper;
using System;
public class QuickstartMovieScraper : WebScraper
{
public override void Init()
{
// Set your license key
License.LicenseKey = "YOUR-LICENSE-KEY";
// Configure scraper settings
this.LoggingLevel = LogLevel.All;
this.WorkingDirectory = @"C:\MovieData\Output\";
// Start scraping from the homepage
this.Request("https://example-movie-site.com", Parse);
}
public override void Parse(Response response)
{
// Extract movie titles using CSS selectors
foreach (var movieDiv in response.Css(".movie-item"))
{
var title = movieDiv.Css("h2")[0].TextContentClean;
var url = movieDiv.Css("a")[0].Attributes["href"];
// Save the scraped data
Scrape(new { Title = title, Url = url }, "movies.json");
}
}
}
// Run the scraper
var scraper = new QuickstartMovieScraper();
scraper.Start();
public class MovieScraper : WebScraper{ public override voidInit() { // Initialize scraper settingsLicense.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; // Request homepage content for scraping this.Request("www.website.com", Parse); } public override voidParse(Response response) { // Iterate over each movie div within the featured movie section foreach (var div in response.Css("#movie-featured > div")) { if (div.Attributes["class"] != "clearfix") { var movieId = Convert.ToInt32(div.GetAttribute("data-movie-id")); var link = div.Css("a")[0]; var movieTitle = link.TextContentClean; // Scrape and store movie data as key-value pairsScrape(new ScrapedData() { { "MovieId", movieId }, { "MovieTitle", movieTitle } }, "Movie.Jsonl"); } } }}
public class MovieScraper : WebScraper
{
public override void Init()
{
// Initialize scraper settings
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
// Request homepage content for scraping
this.Request("www.website.com", Parse);
}
public override void Parse(Response response)
{
// Iterate over each movie div within the featured movie section
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.Attributes["class"] != "clearfix")
{
var movieId = Convert.ToInt32(div.GetAttribute("data-movie-id"));
var link = div.Css("a")[0];
var movieTitle = link.TextContentClean;
// Scrape and store movie data as key-value pairs
Scrape(new ScrapedData()
{
{ "MovieId", movieId },
{ "MovieTitle", movieTitle }
}, "Movie.Jsonl");
}
}
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init() ' Initialize scraper settingsLicense.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\" ' Request homepage content for scrapingMe.Request("www.website.com", AddressOfParse) End Sub PublicOverrides Sub Parse(ByValresponseAsResponse) ' Iterate over each movie div within the featured movie section For Each div Inresponse.Css("#movie-featured > div") If div.Attributes("class") <> "clearfix" Then Dim movieId = Convert.ToInt32(div.GetAttribute("data-movie-id")) Dim link = div.Css("a")(0) Dim movieTitle = link.TextContentClean ' Scrape and store movie data as key-value pairsScrape(New ScrapedData() From { {"MovieId", movieId}, {"MovieTitle", movieTitle} }, "Movie.Jsonl") End If Next End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
' Initialize scraper settings
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
' Request homepage content for scraping
Me.Request("www.website.com", AddressOf Parse)
End Sub
Public Overrides Sub Parse(ByVal response As Response)
' Iterate over each movie div within the featured movie section
For Each div In response.Css("#movie-featured > div")
If div.Attributes("class") <> "clearfix" Then
Dim movieId = Convert.ToInt32(div.GetAttribute("data-movie-id"))
Dim link = div.Css("a")(0)
Dim movieTitle = link.TextContentClean
' Scrape and store movie data as key-value pairs
Scrape(New ScrapedData() From {
{"MovieId", movieId},
{"MovieTitle", movieTitle}
}, "Movie.Jsonl")
End If
Next
End Sub
End Class
工作目錄屬性是用來做什麼的？
這段程式碼有什麼新的？
Working Directory屬性設置了所有爬取資料和相關文件的主要工作目錄。 這確保所有輸出文件都組織在一個位置，便於管理大規模的爬取項目。 如果目錄不存在，將自動建立。
public class Movie{ public intId { get; set; } public stringTitle { get; set; } public stringURL { get; set; }}
public class Movie
{
public int Id { get; set; }
public string Title { get; set; }
public string URL { get; set; }
}
Public Class Movie Public Property IdAsInteger Public Property TitleAsString Public Property URLAsStringEnd Class
Public Class Movie
Public Property Id As Integer
Public Property Title As String
Public Property URL As String
End Class
使用型別化物件如何改善資料組織？
更新程式碼以使用型別化ScrapedData字典：
public class MovieScraper : WebScraper{ public override voidInit() { // Initialize scraper settingsLicense.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; // Request homepage content for scraping this.Request("https://website.com/", Parse); } public override voidParse(Response response) { // Iterate over each movie div within the featured movie section foreach (var div in response.Css("#movie-featured > div")) { if (div.Attributes["class"] != "clearfix") { var movie = new Movie {Id = Convert.ToInt32(div.GetAttribute("data-movie-id")) }; var link = div.Css("a")[0]; movie.Title = link.TextContentClean; movie.URL = link.Attributes["href"]; // Scrape and store movie objectScrape(movie, "Movie.Jsonl"); } } }}
public class MovieScraper : WebScraper
{
public override void Init()
{
// Initialize scraper settings
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
// Request homepage content for scraping
this.Request("https://website.com/", Parse);
}
public override void Parse(Response response)
{
// Iterate over each movie div within the featured movie section
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.Attributes["class"] != "clearfix")
{
var movie = new Movie
{
Id = Convert.ToInt32(div.GetAttribute("data-movie-id"))
};
var link = div.Css("a")[0];
movie.Title = link.TextContentClean;
movie.URL = link.Attributes["href"];
// Scrape and store movie object
Scrape(movie, "Movie.Jsonl");
}
}
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init() ' Initialize scraper settingsLicense.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\" ' Request homepage content for scrapingMe.Request("https://website.com/", AddressOfParse) End Sub PublicOverrides Sub Parse(responseAsResponse) ' Iterate over each movie div within the featured movie section For Each div Inresponse.Css("#movie-featured > div") If div.Attributes("class") <> "clearfix" Then Dim movie As New MovieWith { .Id = Convert.ToInt32(div.GetAttribute("data-movie-id")) } Dim link = div.Css("a")(0) movie.Title = link.TextContentClean movie.URL = link.Attributes("href") ' Scrape and store movie objectScrape(movie, "Movie.Jsonl") End If Next End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
' Initialize scraper settings
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
' Request homepage content for scraping
Me.Request("https://website.com/", AddressOf Parse)
End Sub
Public Overrides Sub Parse(response As Response)
' Iterate over each movie div within the featured movie section
For Each div In response.Css("#movie-featured > div")
If div.Attributes("class") <> "clearfix" Then
Dim movie As New Movie With {
.Id = Convert.ToInt32(div.GetAttribute("data-movie-id"))
}
Dim link = div.Css("a")(0)
movie.Title = link.TextContentClean
movie.URL = link.Attributes("href")
' Scrape and store movie object
Scrape(movie, "Movie.Jsonl")
End If
Next
End Sub
End Class
using System.Collections.Generic;public class Movie{ public intId { get; set; } public stringTitle { get; set; } public stringURL { get; set; } public stringDescription { get; set; } public List<string> Genre { get; set; } public List<string> Actor { get; set; }}
using System.Collections.Generic;
public class Movie
{
public int Id { get; set; }
public string Title { get; set; }
public string URL { get; set; }
public string Description { get; set; }
public List<string> Genre { get; set; }
public List<string> Actor { get; set; }
}
ImportsSystem.Collections.GenericPublic Class Movie Public Property Id() AsInteger Public Property Title() AsString Public Property URL() AsString Public Property Description() AsString Public Property Genre() AsList(OfString) Public Property Actor() AsList(OfString)End Class
Imports System.Collections.Generic
Public Class Movie
Public Property Id() As Integer
Public Property Title() As String
Public Property URL() As String
Public Property Description() As String
Public Property Genre() As List(Of String)
Public Property Actor() As List(Of String)
End Class
public class MovieScraper : WebScraper{ public override voidInit() { // Initialize scraper settingsLicense.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; // Request homepage content for scraping this.Request("https://domain/", Parse); } public override voidParse(Response response) { // Iterate over each movie div within the featured movie section foreach (var div in response.Css("#movie-featured > div")) { if (div.Attributes["class"] != "clearfix") { var movie = new Movie {Id = Convert.ToInt32(div.GetAttribute("data-movie-id")) }; var link = div.Css("a")[0]; movie.Title = link.TextContentClean; movie.URL = link.Attributes["href"]; // Request detailed page this.Request(movie.URL, ParseDetails, new MetaData() { { "movie", movie } }); } } } public voidParseDetails(Response response) { // Retrieve movie object from metadata var movie = response.MetaData.Get<Movie>("movie"); var div = response.Css("div.mvic-desc")[0]; // Extract description movie.Description = div.Css("div.desc")[0].TextContentClean; // Extract genres movie.Genre = new List<string>(); // Initialize genre list foreach(var genre in div.Css("div > p > a")) { movie.Genre.Add(genre.TextContentClean); } // Extract actors movie.Actor = new List<string>(); // Initialize actor list foreach (var actor in div.Css("div > p:nth-child(2) > a")) { movie.Actor.Add(actor.TextContentClean); } // Scrape and store detailed movie dataScrape(movie, "Movie.Jsonl"); }}
public class MovieScraper : WebScraper
{
public override void Init()
{
// Initialize scraper settings
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
// Request homepage content for scraping
this.Request("https://domain/", Parse);
}
public override void Parse(Response response)
{
// Iterate over each movie div within the featured movie section
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.Attributes["class"] != "clearfix")
{
var movie = new Movie
{
Id = Convert.ToInt32(div.GetAttribute("data-movie-id"))
};
var link = div.Css("a")[0];
movie.Title = link.TextContentClean;
movie.URL = link.Attributes["href"];
// Request detailed page
this.Request(movie.URL, ParseDetails, new MetaData() { { "movie", movie } });
}
}
}
public void ParseDetails(Response response)
{
// Retrieve movie object from metadata
var movie = response.MetaData.Get<Movie>("movie");
var div = response.Css("div.mvic-desc")[0];
// Extract description
movie.Description = div.Css("div.desc")[0].TextContentClean;
// Extract genres
movie.Genre = new List<string>(); // Initialize genre list
foreach(var genre in div.Css("div > p > a"))
{
movie.Genre.Add(genre.TextContentClean);
}
// Extract actors
movie.Actor = new List<string>(); // Initialize actor list
foreach (var actor in div.Css("div > p:nth-child(2) > a"))
{
movie.Actor.Add(actor.TextContentClean);
}
// Scrape and store detailed movie data
Scrape(movie, "Movie.Jsonl");
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init() ' Initialize scraper settingsLicense.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\" ' Request homepage content for scrapingMe.Request("https://domain/", AddressOfParse) End Sub PublicOverrides Sub Parse(responseAsResponse) ' Iterate over each movie div within the featured movie section For Each div Inresponse.Css("#movie-featured > div") If div.Attributes("class") <> "clearfix" Then Dim movie As New MovieWith { .Id = Convert.ToInt32(div.GetAttribute("data-movie-id")) } Dim link = div.Css("a")(0) movie.Title = link.TextContentClean movie.URL = link.Attributes("href") ' Request detailed pageMe.Request(movie.URL, AddressOfParseDetails, New MetaData() From {{"movie", movie}}) End If Next End Sub Public Sub ParseDetails(responseAsResponse) ' Retrieve movie object from metadata Dim movie = response.MetaData.Get(OfMovie)("movie") Dim div = response.Css("div.mvic-desc")(0) ' Extract description movie.Description = div.Css("div.desc")(0).TextContentClean ' Extract genres movie.Genre = New List(OfString)() ' Initialize genre list For Each genre In div.Css("div > p > a") movie.Genre.Add(genre.TextContentClean) Next ' Extract actors movie.Actor = New List(OfString)() ' Initialize actor list For Each actor In div.Css("div > p:nth-child(2) > a") movie.Actor.Add(actor.TextContentClean) Next ' Scrape and store detailed movie dataScrape(movie, "Movie.Jsonl") End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
' Initialize scraper settings
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
' Request homepage content for scraping
Me.Request("https://domain/", AddressOf Parse)
End Sub
Public Overrides Sub Parse(response As Response)
' Iterate over each movie div within the featured movie section
For Each div In response.Css("#movie-featured > div")
If div.Attributes("class") <> "clearfix" Then
Dim movie As New Movie With {
.Id = Convert.ToInt32(div.GetAttribute("data-movie-id"))
}
Dim link = div.Css("a")(0)
movie.Title = link.TextContentClean
movie.URL = link.Attributes("href")
' Request detailed page
Me.Request(movie.URL, AddressOf ParseDetails, New MetaData() From {{"movie", movie}})
End If
Next
End Sub
Public Sub ParseDetails(response As Response)
' Retrieve movie object from metadata
Dim movie = response.MetaData.Get(Of Movie)("movie")
Dim div = response.Css("div.mvic-desc")(0)
' Extract description
movie.Description = div.Css("div.desc")(0).TextContentClean
' Extract genres
movie.Genre = New List(Of String)() ' Initialize genre list
For Each genre In div.Css("div > p > a")
movie.Genre.Add(genre.TextContentClean)
Next
' Extract actors
movie.Actor = New List(Of String)() ' Initialize actor list
For Each actor In div.Css("div > p:nth-child(2) > a")
movie.Actor.Add(actor.TextContentClean)
Next
' Scrape and store detailed movie data
Scrape(movie, "Movie.Jsonl")
End Sub
End Class
How does IronWebScraper improve the management of output files?
IronWebScraper improves management by using a working directory to organize all output files from scraping operations in a single, predefined location, facilitating large-scale scraping projects.
What additional data can IronWebScraper extract from detailed movie pages?
IronWebScraper can extract additional data such as movie descriptions, genres, and actors from detailed movie pages, using CSS selectors and custom scraping functions to gather comprehensive data.
