C#으로 블로그를 스크래핑하는 방법

C# 또는 VB .NET 사용하여 Iron WebScraper로 블로그 콘텐츠를 추출해 보겠습니다.

이 튜토리얼에서는 .NET 사용하여 WordPress 블로그(또는 유사한 웹사이트)에서 콘텐츠를 스크래핑하는 방법을 보여줍니다.

// Define a class that extends WebScraper from IronWebScraper
public class BlogScraper : WebScraper
{
    /// <summary>
    /// Override this method to initialize your web-scraper.
    /// Set at least one start URL and configure domain or URL patterns.
    /// </summary>
    public override void Init()
    {
        // Set your license key for IronWebScraper
        License.LicenseKey = "YourLicenseKey";

        // Enable logging for all actions
        this.LoggingLevel = WebScraper.LogLevel.All;

        // Set a directory to store output and cache files
        this.WorkingDirectory = AppSetting.GetAppRoot() + @"\BlogSample\Output\";

        // Enable caching with a specific duration
        EnableWebCache(new TimeSpan(1, 30, 30));

        // Request the start URL and specify the response handler
        this.Request("http://blogSite.com/", Parse);
    }
}

// Define a class that extends WebScraper from IronWebScraper
public class BlogScraper : WebScraper
{
    /// <summary>
    /// Override this method to initialize your web-scraper.
    /// Set at least one start URL and configure domain or URL patterns.
    /// </summary>
    public override void Init()
    {
        // Set your license key for IronWebScraper
        License.LicenseKey = "YourLicenseKey";

        // Enable logging for all actions
        this.LoggingLevel = WebScraper.LogLevel.All;

        // Set a directory to store output and cache files
        this.WorkingDirectory = AppSetting.GetAppRoot() + @"\BlogSample\Output\";

        // Enable caching with a specific duration
        EnableWebCache(new TimeSpan(1, 30, 30));

        // Request the start URL and specify the response handler
        this.Request("http://blogSite.com/", Parse);
    }
}

평소처럼, 우리는 Scraper를 생성하고 WebScraper 클래스를 상속합니다. 이 경우, 그것은 "BlogScraper"입니다.

모든 출력 및 캐시 파일이 저장될 작업 디렉토리를 "\BlogSample\Output\"로 설정했습니다.

다음으로 웹 캐시를 활성화하여 요청된 페이지를 "WebCache"라는 캐시 폴더에 저장합니다.

이제 구문 분석 함수를 작성해 보겠습니다.

/// <summary>
/// Override this method to handle the Http Response for your web scraper.
/// Add additional methods if you handle multiple page types.
/// </summary>
/// <param name="response">The HTTP Response object to parse.</param>
public override void Parse(Response response)
{
    // Iterate over each link found in the section navigation
    foreach (var link in response.Css("div.section-nav > ul > li > a"))
    {
        switch(link.TextContentClean)
        {
            case "Reviews":
                {
                    // Handle reviews case
                }
                break;
            case "Science":
                {
                    // Handle science case
                }
                break;
            default:
                {
                    // Save the link title to a file
                    Scrape(new ScrapedData() { { "Title", link.TextContentClean } }, "BlogScraper.Jsonl");
                }
                break;
        }
    }
}

/// <summary>
/// Override this method to handle the Http Response for your web scraper.
/// Add additional methods if you handle multiple page types.
/// </summary>
/// <param name="response">The HTTP Response object to parse.</param>
public override void Parse(Response response)
{
    // Iterate over each link found in the section navigation
    foreach (var link in response.Css("div.section-nav > ul > li > a"))
    {
        switch(link.TextContentClean)
        {
            case "Reviews":
                {
                    // Handle reviews case
                }
                break;
            case "Science":
                {
                    // Handle science case
                }
                break;
            default:
                {
                    // Save the link title to a file
                    Scrape(new ScrapedData() { { "Title", link.TextContentClean } }, "BlogScraper.Jsonl");
                }
                break;
        }
    }
}

Parse 메서드 내부에서 우리는 상단 메뉴에서 카테고리 페이지(영화, 과학, 리뷰 등)로 가는 모든 링크를 얻습니다.

그런 다음 링크 범주에 따라 적절한 구문 분석 방법으로 전환합니다.

과학 페이지에 사용할 객체 모델을 준비해 봅시다.

/// <summary>
/// Represents a model for Science Page
/// </summary>
public class ScienceModel
{
    /// <summary>
    /// Gets or sets the title.
    /// </summary>
    public string Title { get; set; }

    /// <summary>
    /// Gets or sets the author.
    /// </summary>
    public string Author { get; set; }

    /// <summary>
    /// Gets or sets the date.
    /// </summary>
    public string Date { get; set; }

    /// <summary>
    /// Gets or sets the image.
    /// </summary>
    public string Image { get; set; }

    /// <summary>
    /// Gets or sets the text.
    /// </summary>
    public string Text { get; set; }
}

/// <summary>
/// Represents a model for Science Page
/// </summary>
public class ScienceModel
{
    /// <summary>
    /// Gets or sets the title.
    /// </summary>
    public string Title { get; set; }

    /// <summary>
    /// Gets or sets the author.
    /// </summary>
    public string Author { get; set; }

    /// <summary>
    /// Gets or sets the date.
    /// </summary>
    public string Date { get; set; }

    /// <summary>
    /// Gets or sets the image.
    /// </summary>
    public string Image { get; set; }

    /// <summary>
    /// Gets or sets the text.
    /// </summary>
    public string Text { get; set; }
}

이제 단일 페이지 스크래핑을 구현해 보겠습니다.

/// <summary>
/// Parses the reviews from the response.
/// </summary>
/// <param name="response">The HTTP Response object.</param>
public void ParseReviews(Response response)
{
    // A list to hold Science models
    var scienceList = new List<ScienceModel>();

    foreach (var postBox in response.Css("section.main > div > div.post-list"))
    {
        var item = new ScienceModel
        {
            Title = postBox.Css("h1.headline > a")[0].TextContentClean,
            Author = postBox.Css("div.author > a")[0].TextContentClean,
            Date = postBox.Css("div.time > a")[0].TextContentClean,
            Image = postBox.Css("div.image-wrapper.default-state > img")[0].Attributes["src"],
            Text = postBox.Css("div.summary > p")[0].TextContentClean
        };

        scienceList.Add(item);
    }

    // Save the science list to a JSONL file
    Scrape(scienceList, "BlogScience.Jsonl");
}

/// <summary>
/// Parses the reviews from the response.
/// </summary>
/// <param name="response">The HTTP Response object.</param>
public void ParseReviews(Response response)
{
    // A list to hold Science models
    var scienceList = new List<ScienceModel>();

    foreach (var postBox in response.Css("section.main > div > div.post-list"))
    {
        var item = new ScienceModel
        {
            Title = postBox.Css("h1.headline > a")[0].TextContentClean,
            Author = postBox.Css("div.author > a")[0].TextContentClean,
            Date = postBox.Css("div.time > a")[0].TextContentClean,
            Image = postBox.Css("div.image-wrapper.default-state > img")[0].Attributes["src"],
            Text = postBox.Css("div.summary > p")[0].TextContentClean
        };

        scienceList.Add(item);
    }

    // Save the science list to a JSONL file
    Scrape(scienceList, "BlogScience.Jsonl");
}

모델을 생성한 후에는 응답 객체를 분석하여 주요 요소(제목, 작성자, 날짜, 이미지, 텍스트)를 자세히 살펴볼 수 있습니다.

그런 다음, 우리는 Scrape(object, fileName)를 사용하여 별도의 파일에 결과를 저장합니다.

IronWebscraper 사용법에 대한 전체 튜토리얼을 보려면 여기를 클릭하십시오.

IronWebscraper 시작하기