IronWebScraper通过解析HTML元素提取网站上的电影数据，创建用于结构化数据存储的类型化对象，并使用元数据在页面之间导航以构建全面的电影信息数据集。 该 C# Web Scraper 库简化了将非结构化 Web 内容转换为有组织、可分析数据的过程。
快速入门：在C#中爬取电影
通过NuGet包管理器安装IronWebScraper。
创建一个继承自WebScraper的类。
覆盖Init()以设置许可证并请求目标URL。
覆盖Parse()以使用CSS选择器提取电影数据。
使用Scrape()方法以JSON格式保存数据。
1Install IronWebScraper with NuGet Package Manager
PM > Install-Package IronWebScraper
Install-Package IronWebScraper
2复制并运行这段代码。
using IronWebScraper;using System;public class QuickstartMovieScraper : WebScraper{ public override voidInit() { // Set your license keyLicense.LicenseKey = "YOUR-LICENSE-KEY"; // Configure scraper settings this.LoggingLevel = LogLevel.All; this.WorkingDirectory = @"C:\MovieData\Output\"; // Start scraping from the homepage this.Request("https://example-movie-site.com", Parse); } public override voidParse(Response response) { // Extract movie titles using CSS selectors foreach (var movieDiv in response.Css(".movie-item")) { var title = movieDiv.Css("h2")[0].TextContentClean; var url = movieDiv.Css("a")[0].Attributes["href"]; // Save the scraped dataScrape(new { Title = title, Url = url }, "movies.json"); } }}// Run the scrapervar scraper = new QuickstartMovieScraper();scraper.Start();
using IronWebScraper;
using System;
public class QuickstartMovieScraper : WebScraper
{
public override void Init()
{
// Set your license key
License.LicenseKey = "YOUR-LICENSE-KEY";
// Configure scraper settings
this.LoggingLevel = LogLevel.All;
this.WorkingDirectory = @"C:\MovieData\Output\";
// Start scraping from the homepage
this.Request("https://example-movie-site.com", Parse);
}
public override void Parse(Response response)
{
// Extract movie titles using CSS selectors
foreach (var movieDiv in response.Css(".movie-item"))
{
var title = movieDiv.Css("h2")[0].TextContentClean;
var url = movieDiv.Css("a")[0].Attributes["href"];
// Save the scraped data
Scrape(new { Title = title, Url = url }, "movies.json");
}
}
}
// Run the scraper
var scraper = new QuickstartMovieScraper();
scraper.Start();
public class MovieScraper : WebScraper{ public override voidInit() { // Initialize scraper settingsLicense.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; // Request homepage content for scraping this.Request("www.website.com", Parse); } public override voidParse(Response response) { // Iterate over each movie div within the featured movie section foreach (var div in response.Css("#movie-featured > div")) { if (div.Attributes["class"] != "clearfix") { var movieId = Convert.ToInt32(div.GetAttribute("data-movie-id")); var link = div.Css("a")[0]; var movieTitle = link.TextContentClean; // Scrape and store movie data as key-value pairsScrape(new ScrapedData() { { "MovieId", movieId }, { "MovieTitle", movieTitle } }, "Movie.Jsonl"); } } }}
public class MovieScraper : WebScraper
{
public override void Init()
{
// Initialize scraper settings
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
// Request homepage content for scraping
this.Request("www.website.com", Parse);
}
public override void Parse(Response response)
{
// Iterate over each movie div within the featured movie section
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.Attributes["class"] != "clearfix")
{
var movieId = Convert.ToInt32(div.GetAttribute("data-movie-id"));
var link = div.Css("a")[0];
var movieTitle = link.TextContentClean;
// Scrape and store movie data as key-value pairs
Scrape(new ScrapedData()
{
{ "MovieId", movieId },
{ "MovieTitle", movieTitle }
}, "Movie.Jsonl");
}
}
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init() ' Initialize scraper settingsLicense.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\" ' Request homepage content for scrapingMe.Request("www.website.com", AddressOfParse) End Sub PublicOverrides Sub Parse(ByValresponseAsResponse) ' Iterate over each movie div within the featured movie section For Each div Inresponse.Css("#movie-featured > div") If div.Attributes("class") <> "clearfix" Then Dim movieId = Convert.ToInt32(div.GetAttribute("data-movie-id")) Dim link = div.Css("a")(0) Dim movieTitle = link.TextContentClean ' Scrape and store movie data as key-value pairsScrape(New ScrapedData() From { {"MovieId", movieId}, {"MovieTitle", movieTitle} }, "Movie.Jsonl") End If Next End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
' Initialize scraper settings
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
' Request homepage content for scraping
Me.Request("www.website.com", AddressOf Parse)
End Sub
Public Overrides Sub Parse(ByVal response As Response)
' Iterate over each movie div within the featured movie section
For Each div In response.Css("#movie-featured > div")
If div.Attributes("class") <> "clearfix" Then
Dim movieId = Convert.ToInt32(div.GetAttribute("data-movie-id"))
Dim link = div.Css("a")(0)
Dim movieTitle = link.TextContentClean
' Scrape and store movie data as key-value pairs
Scrape(New ScrapedData() From {
{"MovieId", movieId},
{"MovieTitle", movieTitle}
}, "Movie.Jsonl")
End If
Next
End Sub
End Class
工作目录属性有什么用？
本代码有哪些新内容？
Working Directory属性设置所有抓取数据和相关文件的主工作目录。 这样可以确保所有输出文件都组织在一个位置，从而更便于管理大规模的刮擦项目。 如果目录不存在，将自动创建。
何时应使用 CSS 选择器与属性？
其他注意事项：
CSS 选择器是通过结构位置或类名来定位元素的理想选择，而直接属性访问则更适合提取 ID 或自定义数据属性等特定值。 在我们的示例中，我们使用CSS选择器（data-movie-id）以提取特定值。
public class Movie{ public intId { get; set; } public stringTitle { get; set; } public stringURL { get; set; }}
public class Movie
{
public int Id { get; set; }
public string Title { get; set; }
public string URL { get; set; }
}
Public Class Movie Public Property IdAsInteger Public Property TitleAsString Public Property URLAsStringEnd Class
Public Class Movie
Public Property Id As Integer
Public Property Title As String
Public Property URL As String
End Class
使用类型对象如何改进数据组织？
更新代码以使用类型化的ScrapedData字典：
public class MovieScraper : WebScraper{ public override voidInit() { // Initialize scraper settingsLicense.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; // Request homepage content for scraping this.Request("https://website.com/", Parse); } public override voidParse(Response response) { // Iterate over each movie div within the featured movie section foreach (var div in response.Css("#movie-featured > div")) { if (div.Attributes["class"] != "clearfix") { var movie = new Movie {Id = Convert.ToInt32(div.GetAttribute("data-movie-id")) }; var link = div.Css("a")[0]; movie.Title = link.TextContentClean; movie.URL = link.Attributes["href"]; // Scrape and store movie objectScrape(movie, "Movie.Jsonl"); } } }}
public class MovieScraper : WebScraper
{
public override void Init()
{
// Initialize scraper settings
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
// Request homepage content for scraping
this.Request("https://website.com/", Parse);
}
public override void Parse(Response response)
{
// Iterate over each movie div within the featured movie section
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.Attributes["class"] != "clearfix")
{
var movie = new Movie
{
Id = Convert.ToInt32(div.GetAttribute("data-movie-id"))
};
var link = div.Css("a")[0];
movie.Title = link.TextContentClean;
movie.URL = link.Attributes["href"];
// Scrape and store movie object
Scrape(movie, "Movie.Jsonl");
}
}
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init() ' Initialize scraper settingsLicense.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\" ' Request homepage content for scrapingMe.Request("https://website.com/", AddressOfParse) End Sub PublicOverrides Sub Parse(responseAsResponse) ' Iterate over each movie div within the featured movie section For Each div Inresponse.Css("#movie-featured > div") If div.Attributes("class") <> "clearfix" Then Dim movie As New MovieWith { .Id = Convert.ToInt32(div.GetAttribute("data-movie-id")) } Dim link = div.Css("a")(0) movie.Title = link.TextContentClean movie.URL = link.Attributes("href") ' Scrape and store movie objectScrape(movie, "Movie.Jsonl") End If Next End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
' Initialize scraper settings
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
' Request homepage content for scraping
Me.Request("https://website.com/", AddressOf Parse)
End Sub
Public Overrides Sub Parse(response As Response)
' Iterate over each movie div within the featured movie section
For Each div In response.Css("#movie-featured > div")
If div.Attributes("class") <> "clearfix" Then
Dim movie As New Movie With {
.Id = Convert.ToInt32(div.GetAttribute("data-movie-id"))
}
Dim link = div.Css("a")(0)
movie.Title = link.TextContentClean
movie.URL = link.Attributes("href")
' Scrape and store movie object
Scrape(movie, "Movie.Jsonl")
End If
Next
End Sub
End Class
using System.Collections.Generic;public class Movie{ public intId { get; set; } public stringTitle { get; set; } public stringURL { get; set; } public stringDescription { get; set; } public List<string> Genre { get; set; } public List<string> Actor { get; set; }}
using System.Collections.Generic;
public class Movie
{
public int Id { get; set; }
public string Title { get; set; }
public string URL { get; set; }
public string Description { get; set; }
public List<string> Genre { get; set; }
public List<string> Actor { get; set; }
}
ImportsSystem.Collections.GenericPublic Class Movie Public Property Id() AsInteger Public Property Title() AsString Public Property URL() AsString Public Property Description() AsString Public Property Genre() AsList(OfString) Public Property Actor() AsList(OfString)End Class
Imports System.Collections.Generic
Public Class Movie
Public Property Id() As Integer
Public Property Title() As String
Public Property URL() As String
Public Property Description() As String
Public Property Genre() As List(Of String)
Public Property Actor() As List(Of String)
End Class
public class MovieScraper : WebScraper{ public override voidInit() { // Initialize scraper settingsLicense.LicenseKey = "LicenseKey"; this.LoggingLevel = WebScraper.LogLevel.All; this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\"; // Request homepage content for scraping this.Request("https://domain/", Parse); } public override voidParse(Response response) { // Iterate over each movie div within the featured movie section foreach (var div in response.Css("#movie-featured > div")) { if (div.Attributes["class"] != "clearfix") { var movie = new Movie {Id = Convert.ToInt32(div.GetAttribute("data-movie-id")) }; var link = div.Css("a")[0]; movie.Title = link.TextContentClean; movie.URL = link.Attributes["href"]; // Request detailed page this.Request(movie.URL, ParseDetails, new MetaData() { { "movie", movie } }); } } } public voidParseDetails(Response response) { // Retrieve movie object from metadata var movie = response.MetaData.Get<Movie>("movie"); var div = response.Css("div.mvic-desc")[0]; // Extract description movie.Description = div.Css("div.desc")[0].TextContentClean; // Extract genres movie.Genre = new List<string>(); // Initialize genre list foreach(var genre in div.Css("div > p > a")) { movie.Genre.Add(genre.TextContentClean); } // Extract actors movie.Actor = new List<string>(); // Initialize actor list foreach (var actor in div.Css("div > p:nth-child(2) > a")) { movie.Actor.Add(actor.TextContentClean); } // Scrape and store detailed movie dataScrape(movie, "Movie.Jsonl"); }}
public class MovieScraper : WebScraper
{
public override void Init()
{
// Initialize scraper settings
License.LicenseKey = "LicenseKey";
this.LoggingLevel = WebScraper.LogLevel.All;
this.WorkingDirectory = AppSetting.GetAppRoot() + @"\MovieSample\Output\";
// Request homepage content for scraping
this.Request("https://domain/", Parse);
}
public override void Parse(Response response)
{
// Iterate over each movie div within the featured movie section
foreach (var div in response.Css("#movie-featured > div"))
{
if (div.Attributes["class"] != "clearfix")
{
var movie = new Movie
{
Id = Convert.ToInt32(div.GetAttribute("data-movie-id"))
};
var link = div.Css("a")[0];
movie.Title = link.TextContentClean;
movie.URL = link.Attributes["href"];
// Request detailed page
this.Request(movie.URL, ParseDetails, new MetaData() { { "movie", movie } });
}
}
}
public void ParseDetails(Response response)
{
// Retrieve movie object from metadata
var movie = response.MetaData.Get<Movie>("movie");
var div = response.Css("div.mvic-desc")[0];
// Extract description
movie.Description = div.Css("div.desc")[0].TextContentClean;
// Extract genres
movie.Genre = new List<string>(); // Initialize genre list
foreach(var genre in div.Css("div > p > a"))
{
movie.Genre.Add(genre.TextContentClean);
}
// Extract actors
movie.Actor = new List<string>(); // Initialize actor list
foreach (var actor in div.Css("div > p:nth-child(2) > a"))
{
movie.Actor.Add(actor.TextContentClean);
}
// Scrape and store detailed movie data
Scrape(movie, "Movie.Jsonl");
}
}
Public Class MovieScraperInheritsWebScraper PublicOverrides Sub Init() ' Initialize scraper settingsLicense.LicenseKey = "LicenseKey"Me.LoggingLevel = WebScraper.LogLevel.AllMe.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\" ' Request homepage content for scrapingMe.Request("https://domain/", AddressOfParse) End Sub PublicOverrides Sub Parse(responseAsResponse) ' Iterate over each movie div within the featured movie section For Each div Inresponse.Css("#movie-featured > div") If div.Attributes("class") <> "clearfix" Then Dim movie As New MovieWith { .Id = Convert.ToInt32(div.GetAttribute("data-movie-id")) } Dim link = div.Css("a")(0) movie.Title = link.TextContentClean movie.URL = link.Attributes("href") ' Request detailed pageMe.Request(movie.URL, AddressOfParseDetails, New MetaData() From {{"movie", movie}}) End If Next End Sub Public Sub ParseDetails(responseAsResponse) ' Retrieve movie object from metadata Dim movie = response.MetaData.Get(OfMovie)("movie") Dim div = response.Css("div.mvic-desc")(0) ' Extract description movie.Description = div.Css("div.desc")(0).TextContentClean ' Extract genres movie.Genre = New List(OfString)() ' Initialize genre list For Each genre In div.Css("div > p > a") movie.Genre.Add(genre.TextContentClean) Next ' Extract actors movie.Actor = New List(OfString)() ' Initialize actor list For Each actor In div.Css("div > p:nth-child(2) > a") movie.Actor.Add(actor.TextContentClean) Next ' Scrape and store detailed movie dataScrape(movie, "Movie.Jsonl") End SubEnd Class
Public Class MovieScraper
Inherits WebScraper
Public Overrides Sub Init()
' Initialize scraper settings
License.LicenseKey = "LicenseKey"
Me.LoggingLevel = WebScraper.LogLevel.All
Me.WorkingDirectory = AppSetting.GetAppRoot() & "\MovieSample\Output\"
' Request homepage content for scraping
Me.Request("https://domain/", AddressOf Parse)
End Sub
Public Overrides Sub Parse(response As Response)
' Iterate over each movie div within the featured movie section
For Each div In response.Css("#movie-featured > div")
If div.Attributes("class") <> "clearfix" Then
Dim movie As New Movie With {
.Id = Convert.ToInt32(div.GetAttribute("data-movie-id"))
}
Dim link = div.Css("a")(0)
movie.Title = link.TextContentClean
movie.URL = link.Attributes("href")
' Request detailed page
Me.Request(movie.URL, AddressOf ParseDetails, New MetaData() From {{"movie", movie}})
End If
Next
End Sub
Public Sub ParseDetails(response As Response)
' Retrieve movie object from metadata
Dim movie = response.MetaData.Get(Of Movie)("movie")
Dim div = response.Css("div.mvic-desc")(0)
' Extract description
movie.Description = div.Css("div.desc")(0).TextContentClean
' Extract genres
movie.Genre = New List(Of String)() ' Initialize genre list
For Each genre In div.Css("div > p > a")
movie.Genre.Add(genre.TextContentClean)
Next
' Extract actors
movie.Actor = New List(Of String)() ' Initialize actor list
For Each actor In div.Css("div > p:nth-child(2) > a")
movie.Actor.Add(actor.TextContentClean)
Next
' Scrape and store detailed movie data
Scrape(movie, "Movie.Jsonl")
End Sub
End Class
How does IronWebScraper improve the management of output files?
IronWebScraper improves management by using a working directory to organize all output files from scraping operations in a single, predefined location, facilitating large-scale scraping projects.
What additional data can IronWebScraper extract from detailed movie pages?
IronWebScraper can extract additional data such as movie descriptions, genres, and actors from detailed movie pages, using CSS selectors and custom scraping functions to gather comprehensive data.
