-I am using selenium and PhantomJSDriver for web scrapping.I am facing the issue for scrapping list of items for wbsite.The result is not consistent,I get different number of records each time.To overcome this,I use explicit and implicit wait for webdriver.But it through timeout error for that case.
driver.FindElementsByClassName("Tile").ToList()
retrive different number of records.and if I use debugger and wait for some time then it retrive all records.
- driver.Url = uri.ToString() ;
- driver.Navigate();
- if (uri.PathAndQuery == "/")
- {
-
- driver.FindElementsByClassName("Tile").ToList().ForEach(x =>
- {
- objUPCProcess = new UPCProcessingModel();
-
- try
- {
- objUPCProcess.Description = x.FindElement(By.ClassName("Tile-heading")).Text;
- objUPCProcess.Price = x.FindElement(By.ClassName("Price-display")).Text;
- listOfProductDetails.Add(objUPCProcess);
- if (i == 0)
- {
- log.Item = objUPCProcess.Description;
- i++;
- }
- }
- catch (OpenQA.Selenium.NoSuchElementException ex)
- {
- log.Message = ex.Message;
- log.Status = "Error";
- }
- });
- }