skip to Main Content

I have a selenium scraper that open the site and login in to it. The site has been written by react or react native and the data of the page get from API.
I want when the selenium open the browser and login to it capture the Get and Post Request too.
I want to get the specific header from one of the API request.

I have tried many way such as browsermobproxy (not compatible with .net 6 ) and other ways but no one can’t solve my problem.
I have tried this too :

ChromeOptions options = new ChromeOptions();
options.AddArgument("user-data-dir=C:\Users\Test\AppData\Local\Google\Chrome\User Data");

// Create a new ChromeDriver object.
IWebDriver driver = new ChromeDriver(options);

// Navigate to the target website.
driver.Navigate().GoToUrl("https://test.com/");

// Get the network requests.
List<NetworkRequest> requests = driver.GetNetworkRequests();

// Print the request details to the console.
foreach (NetworkRequest request in requests)
{
    Console.WriteLine($"Request URL: {request.Url}");
    Console.WriteLine($"Request Method: {request.Method}");
    Console.WriteLine($"Request Status Code: {request.StatusCode}");
    Console.WriteLine($"Request Headers:");
    foreach (var header in request.Headers)
    {
        Console.WriteLine($"  {header.Key}: {header.Value}");
    }
    Console.WriteLine();
}

// Close the browser.
driver.Quit();

2

Answers


  1. Chosen as BEST ANSWER

    I Use this and it solve my problem

    ChromeOptions options = new ChromeOptions();
    options.SetLoggingPreference("performance", LogLevel.All);
    options.AddArgument("user-data-dir=C:\Users\Test\AppData\Local\Google\Chrome\User Data");
    IWebDriver driver = new ChromeDriver(options);
    driver.Navigate().GoToUrl("https://test.com/");
    var performanceLogs = driver.Manage().Logs.GetLog("performance");
    // Find Specific Request
    var jsonText = performanceLogs.Where(p => p.Message.Contains("some of the api url or the header name")).FirstOrDefault();
    //Header
    string pattern = ""The Header Name":"([^"]+)"";
    Match match = Regex.Match(jsonText.ToString(), pattern);
    string Header= "";
    if (match.Success)
    {
    Header= match.Groups[1].Value;
    Console.WriteLine("Header: " + Header);
    }
    else
    {
    Console.WriteLine("Header not found.");
    }
    driver.Quit();
    

  2. You can use devtools with chrome driver

    using var driver = new ChromeDriver();
    
    IDevTools devTools = driver;
    DevToolsSession session = devTools.GetDevToolsSession();
    await session.Domains.Network.EnableNetwork();
    
    session.DevToolsEventReceived += OnDevToolsEventReceived;
    
    void OnDevToolsEventReceived(object? sender, DevToolsEventReceivedEventArgs e)
    {
        if (e.EventName == "requestWillBeSentExtraInfo")
        {
            var requestHeaders = e.EventData["headers"];
        }
        if (e.EventName == "responseReceivedExtraInfo")
        {
            var responseHeaders = e.EventData["headers"];
        }  
    }
    
    driver.Navigate().GoToUrl("https://stackoverflow.com/");
    
    Login or Signup to reply.
Please signup or login to give your own answer.
Back To Top
Search