Web Scraping eines Blogs in C# mit IronWebScraper

// Define a class that extends WebScraper from IronWebScraper
public class BlogScraper : WebScraper
{
    /// <summary>
    /// Override this method to initialize your web-scraper.
    /// Set at least one start URL and configure domain or URL patterns.
    /// </summary>
    public override void Init()
    {
        // Set your license key for IronWebScraper
        License.LicenseKey = "YourLicenseKey";

        // Enable logging for all actions
        this.LoggingLevel = WebScraper.LogLevel.All;

        // Set a directory to store output and cache files
        this.WorkingDirectory = AppSetting.GetAppRoot() + @"\BlogSample\Output\";

        // Enable caching with a specific duration
        EnableWebCache(new TimeSpan(1, 30, 30));

        // Request the start URL and specify the response handler
        this.Request("http://blogSite.com/", Parse);
    }
}

' Define a class that extends WebScraper from IronWebScraper
Public Class BlogScraper
	Inherits WebScraper

	''' <summary>
	''' Override this method to initialize your web-scraper.
	''' Set at least one start URL and configure domain or URL patterns.
	''' </summary>
	Public Overrides Sub Init()
		' Set your license key for IronWebScraper
		License.LicenseKey = "YourLicenseKey"

		' Enable logging for all actions
		Me.LoggingLevel = WebScraper.LogLevel.All

		' Set a directory to store output and cache files
		Me.WorkingDirectory = AppSetting.GetAppRoot() & "\BlogSample\Output\"

		' Enable caching with a specific duration
		EnableWebCache(New TimeSpan(1, 30, 30))

		' Request the start URL and specify the response handler
		Me.Request("http://blogSite.com/", Parse)
	End Sub
End Class

/// <summary>
/// Override this method to handle the Http Response for your web scraper.
/// Add additional methods if you handle multiple page types.
/// </summary>
/// <param name="response">The HTTP Response object to parse.</param>
public override void Parse(Response response)
{
    // Iterate over each link found in the section navigation
    foreach (var link in response.Css("div.section-nav > ul > li > a"))
    {
        switch(link.TextContentClean)
        {
            case "Reviews":
                {
                    // Handle reviews case
                }
                break;
            case "Science":
                {
                    // Handle science case
                }
                break;
            default:
                {
                    // Save the link title to a file
                    Scrape(new ScrapedData() { { "Title", link.TextContentClean } }, "BlogScraper.Jsonl");
                }
                break;
        }
    }
}

''' <summary>
''' Override this method to handle the Http Response for your web scraper.
''' Add additional methods if you handle multiple page types.
''' </summary>
''' <param name="response">The HTTP Response object to parse.</param>
Public Overrides Sub Parse(ByVal response As Response)
	' Iterate over each link found in the section navigation
	For Each link In response.Css("div.section-nav > ul > li > a")
		Select Case link.TextContentClean
			Case "Reviews"
					' Handle reviews case
			Case "Science"
					' Handle science case
			Case Else
					' Save the link title to a file
					Scrape(New ScrapedData() From {
						{ "Title", link.TextContentClean }
					},
					"BlogScraper.Jsonl")
		End Select
	Next link
End Sub

/// <summary>
/// Represents a model for Science Page
/// </summary>
public class ScienceModel
{
    /// <summary>
    /// Gets or sets the title.
    /// </summary>
    public string Title { get; set; }

    /// <summary>
    /// Gets or sets the author.
    /// </summary>
    public string Author { get; set; }

    /// <summary>
    /// Gets or sets the date.
    /// </summary>
    public string Date { get; set; }

    /// <summary>
    /// Gets or sets the image.
    /// </summary>
    public string Image { get; set; }

    /// <summary>
    /// Gets or sets the text.
    /// </summary>
    public string Text { get; set; }
}

''' <summary>
''' Represents a model for Science Page
''' </summary>
Public Class ScienceModel
	''' <summary>
	''' Gets or sets the title.
	''' </summary>
	Public Property Title() As String

	''' <summary>
	''' Gets or sets the author.
	''' </summary>
	Public Property Author() As String

	''' <summary>
	''' Gets or sets the date.
	''' </summary>
	Public Property [Date]() As String

	''' <summary>
	''' Gets or sets the image.
	''' </summary>
	Public Property Image() As String

	''' <summary>
	''' Gets or sets the text.
	''' </summary>
	Public Property Text() As String
End Class

/// <summary>
/// Parses the reviews from the response.
/// </summary>
/// <param name="response">The HTTP Response object.</param>
public void ParseReviews(Response response)
{
    // A list to hold Science models
    var scienceList = new List<ScienceModel>();

    foreach (var postBox in response.Css("section.main > div > div.post-list"))
    {
        var item = new ScienceModel
        {
            Title = postBox.Css("h1.headline > a")[0].TextContentClean,
            Author = postBox.Css("div.author > a")[0].TextContentClean,
            Date = postBox.Css("div.time > a")[0].TextContentClean,
            Image = postBox.Css("div.image-wrapper.default-state > img")[0].Attributes["src"],
            Text = postBox.Css("div.summary > p")[0].TextContentClean
        };

        scienceList.Add(item);
    }

    // Save the science list to a JSONL file
    Scrape(scienceList, "BlogScience.Jsonl");
}

''' <summary>
''' Parses the reviews from the response.
''' </summary>
''' <param name="response">The HTTP Response object.</param>
Public Sub ParseReviews(ByVal response As Response)
	' A list to hold Science models
	Dim scienceList = New List(Of ScienceModel)()

	For Each postBox In response.Css("section.main > div > div.post-list")
		Dim item = New ScienceModel With {
			.Title = postBox.Css("h1.headline > a")(0).TextContentClean,
			.Author = postBox.Css("div.author > a")(0).TextContentClean,
			.Date = postBox.Css("div.time > a")(0).TextContentClean,
			.Image = postBox.Css("div.image-wrapper.default-state > img")(0).Attributes("src"),
			.Text = postBox.Css("div.summary > p")(0).TextContentClean
		}

		scienceList.Add(item)
	Next postBox

	' Save the science list to a JSONL file
	Scrape(scienceList, "BlogScience.Jsonl")
End Sub