C#과 Selenium으로 크롤링, 2024-04-04

sunj - Apr 4 - - Dev Community

Form1.Designer.cs

namespace NaraJanteo_c_
{// Form1.Designer.cs

    partial class Form1
    {
        private System.Windows.Forms.DateTimePicker startDatePicker;
        private System.Windows.Forms.DateTimePicker endDatePicker;
        private System.Windows.Forms.Button crawlButton;

        private void InitializeComponent()
        {
            this.startDatePicker = new System.Windows.Forms.DateTimePicker();
            this.endDatePicker = new System.Windows.Forms.DateTimePicker();
            this.crawlButton = new System.Windows.Forms.Button();
            // 
            // startDatePicker
            // 
            this.startDatePicker.Location = new System.Drawing.Point(50, 50);
            this.startDatePicker.Name = "startDatePicker";
            this.startDatePicker.Size = new System.Drawing.Size(200, 20);
            this.startDatePicker.TabIndex = 0;
            // 
            // endDatePicker
            // 
            this.endDatePicker.Location = new System.Drawing.Point(50, 100);
            this.endDatePicker.Name = "endDatePicker";
            this.endDatePicker.Size = new System.Drawing.Size(200, 20);
            this.endDatePicker.TabIndex = 1;
            // 
            // crawlButton
            // 
            this.crawlButton.Location = new System.Drawing.Point(100, 150);
            this.crawlButton.Name = "crawlButton";
            this.crawlButton.Size = new System.Drawing.Size(100, 30);
            this.crawlButton.Text = "데이터";
            this.crawlButton.UseVisualStyleBackColor = true;
            this.crawlButton.Click += new System.EventHandler(this.crawlButton_Click);
            // 
            // Form1
            // 
            this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
            this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
            this.ClientSize = new System.Drawing.Size(300, 200);
            this.Controls.Add(this.crawlButton);
            this.Controls.Add(this.endDatePicker);
            this.Controls.Add(this.startDatePicker);
            this.Name = "Form1";
            this.Text = "데이터 애플리케이션";
            this.ResumeLayout(false);
        }
    }

}


Enter fullscreen mode Exit fullscreen mode

Form1.cs

using System;
using System.Diagnostics;
using System.Windows.Forms;
using static System.Net.WebRequestMethods;
using OfficeOpenXml;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using System.IO;

namespace NaraJanteo_c_
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        private void crawlButton_Click(object sender, EventArgs e)
        {
            // 사용자가 선택한 시작 및 종료 날짜 가져오기
            DateTime startDate = startDatePicker.Value;
            DateTime endDate = endDatePicker.Value;

            // startDate가 endDate보다 클 경우 경고창을 표시
            if (startDate > endDate)
            {
                MessageBox.Show("시작 날짜가 종료 날짜보다 큽니다. 다시 선택해주세요.", "경고", MessageBoxButtons.OK, MessageBoxIcon.Warning);
                return;
            }

            // 엑셀 파일 경로 선택
            SaveFileDialog saveFileDialog = new SaveFileDialog();
            saveFileDialog.OverwritePrompt = false;
            saveFileDialog.Filter = "Excel 파일 (*.xlsx)|*.xlsx";
            saveFileDialog.Title = "엑셀 파일 저장 위치 선택";
            saveFileDialog.ShowDialog();
            string filePath = saveFileDialog.FileName;

            // 크롤링 및 데이터 저장
            CrawlAndSaveData(startDate, endDate, filePath);

            MessageBox.Show("데이터 저장이 완료되었습니다.");
        }

        private void CrawlAndSaveData(DateTime startDate, DateTime endDate, string filePath)
        {
            ExcelPackage.LicenseContext = LicenseContext.NonCommercial;

            using (IWebDriver driver = new ChromeDriver())
            {
                // 크롤링할 웹사이트의 URL 설정
                string url = "https://www.g2b.go.kr/index.jsp";

                // 웹페이지 열기
                driver.Navigate().GoToUrl(url);

                // 검색창 요소를 찾아서 "rpa"를 입력
                IWebElement searchBox = driver.FindElement(By.XPath("//*[@id=\'bidNm\']"));
                searchBox.SendKeys("rpa");

                var setFromDays = driver.FindElement(By.XPath("//*[@id=\'noticedate\']/button[1]"));
                setFromDays.Click();


                var fiveY = startDate.Year.ToString();
                var fiveM = string.Format("{0:MM}", startDate);
                var IfiveM = int.Parse(fiveM);
                var fiveD = startDate.Day.ToString();
                var IfiveY = int.Parse(fiveY);

                var aMonth = driver.FindElement(By.XPath("//*[@id=\"ui-datepicker-div\"]/div[1]/div/span[2]")).Text;
                var aYear = driver.FindElement(By.XPath("//*[@id=\'ui-datepicker-div\']/div[1]/div/span[1]")).Text;

                int iMonth = 0;
                int iYear = 0;

                for (int i = 0; i < aMonth.Length; i++)
                {
                    iMonth = iMonth * 10 + (aMonth[i] - '0');
                }

                for (int i = 0; i < aYear.Length; i++)
                {
                    iYear = iYear * 10 + (aYear[i] - '0');
                }

                if (iMonth > IfiveM) {
                    while (!(aMonth.Equals(fiveM) && aYear.Equals(fiveY)))
                    {
                        driver.FindElement(By.XPath("//*[@id=\'datepicker-prev\']")).Click();
                        aMonth = driver.FindElement(By.XPath("//*[@id=\"ui-datepicker-div\"]/div[1]/div/span[2]")).Text;
                        aYear = driver.FindElement(By.XPath("//*[@id=\'ui-datepicker-div\']/div[1]/div/span[1]")).Text;
                    }
                }if (iMonth < IfiveM){
                    while (!(aMonth.Equals(fiveM) && aYear.Equals(fiveY)))
                    {
                        driver.FindElement(By.ClassName("ui-datepicker-next")).Click();
                        aMonth = driver.FindElement(By.XPath("//*[@id=\"ui-datepicker-div\"]/div[1]/div/span[2]")).Text;
                        aYear = driver.FindElement(By.XPath("//*[@id=\'ui-datepicker-div\']/div[1]/div/span[1]")).Text;
                    }
                }

                Debug.WriteLine("시작 날짜 ::: " + fiveY + "/" + fiveM + "/" + fiveD);

                IWebElement calandar = driver.FindElement(By.XPath("//*[@id=\'ui-datepicker-div\']/table"));
                IWebElement tbody = calandar.FindElement(By.TagName("tbody"));
                var trs = tbody.FindElements(By.TagName("tr"));

                foreach (var tr in trs)
                  {
                     var tds = tr.FindElements(By.TagName("td"));
                     foreach (var td in tds)
                     {
                        if (td.Text.Equals(fiveD))
                         {
                            td.Click();
                         }
                      }
                }

                IWebElement searchButton = driver.FindElement(By.XPath("//*[@id=\'searchForm\']/div/fieldset[1]/ul/li[4]/dl/dd[3]/a"));

                // 검색 버튼 클릭
                searchButton.Click();

                Debug.WriteLine("____________________검색_____________________");

                // 페이지가 로드될 때까지 대기 (예: 10초)
                driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(10);

                try
                {
                    IWebElement firstFrame = driver.FindElement(By.XPath("//*[@id=\'sub\']"));
                    driver.SwitchTo().Frame(firstFrame); // 첫 번째 frame으로 전환
                    driver.SwitchTo().Frame("main"); // 두 번째 frame으로 전환

                    IWebElement table = driver.FindElement(By.XPath("//*[@id=\'resultForm\']/div[2]/table"));

                    IWebElement Rtbody = table.FindElement(By.TagName("tbody"));
                    var Rtrs = Rtbody.FindElements(By.TagName("tr"));

                    //string fileName = "장표.xlsx";
                    //string path = "C:\\Users\\Public\\Documents\\";

                    string[] data = { "공고명", "검색 키워드", "공고기관", "수요기관", "입력일시", "검색일시" };

                    bool fileExists = System.IO.File.Exists(filePath);
                    FileInfo fi = new FileInfo(filePath);


                    using (ExcelPackage pck = new ExcelPackage(fi))
                    {
                        ExcelWorksheet wsheet;

                        if (fileExists)
                        {
                            wsheet = pck.Workbook.Worksheets["Sheet1"];
                        }
                        else {
                            wsheet = pck.Workbook.Worksheets.Add("Sheet1");

                            for (int k = 0; k < data.Length; k++)
                            {
                                wsheet.Cells[1, k + 1].Value = data[k];
                            }
                        }
                        int lastRow = wsheet.Dimension.End.Row + 1;

                        for (int j = 0; j < Rtrs.Count; j++){
                           var tds = Rtrs[j].FindElements(By.TagName("td"));
                           for (int i = 3; i < tds.Count - 2; i++)
                           {
                              var div = tds[i].FindElement(By.TagName("div"));
                              wsheet.Cells[lastRow, i - 2].Value = div.Text;
                           }
                           wsheet.Cells[lastRow, data.Length].Value = DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss");
                         }
                         pck.Save();

                            Debug.WriteLine("엑셀 저장 성공");
                        }


                    }
                catch (NoSuchElementException e)
                {
                    Debug.WriteLine(e.ToString() + "::: error!_____________________");
                }
                catch (NoSuchFrameException ne)
                {
                    Debug.WriteLine(ne.ToString() + "::: error!_____________________");
                }

                // WebDriver 종료
                driver.Quit();
            }

    }
    }
}

Enter fullscreen mode Exit fullscreen mode
. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
Terabox Video Player