github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-06-08 00:27:32 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Make Rubocop happy

Browse Source
This commit is contained in:
Gerhard Schlager
2018-10-31 01:30:14 +01:00
parent 65db9326b4
commit d6f89a85ef
1 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
script/import_scripts/google_groups.rb
View File

@ -48,20 +48,20 @@ def get(url)
end end
end end
def extract(css, elem = driver) def extract(css, parent_element = driver)
begin begin
retries ||= 0 retries ||= 0
elem.find_elements(css: css).map { |elem| yield(elem) } parent_element.find_elements(css: css).map { |element| yield(element) }
rescue Net::ReadTimeout, Selenium::WebDriver::Error::StaleElementReferenceError rescue Net::ReadTimeout, Selenium::WebDriver::Error::StaleElementReferenceError
sleep retries sleep retries
retry if (retries += 1) < MAX_FIND_RETRIES retry if (retries += 1) < MAX_FIND_RETRIES
end end
end end
def find(css, elem = driver) def find(css, parent_element = driver)
begin begin
retries ||= 0 retries ||= 0
elem.find_element(css: css) parent_element.find_element(css: css)
rescue Net::ReadTimeout, Selenium::WebDriver::Error::ElementNotVisibleError rescue Net::ReadTimeout, Selenium::WebDriver::Error::ElementNotVisibleError
sleep retries sleep retries
retry if (retries += 1) < MAX_FIND_RETRIES retry if (retries += 1) < MAX_FIND_RETRIES
@ -73,10 +73,10 @@ def crawl_categories
url = "https://groups.google.com/forum/?_escaped_fragment_=categories/#{@groupname}[#{start}-#{start + 99}]" url = "https://groups.google.com/forum/?_escaped_fragment_=categories/#{@groupname}[#{start}-#{start + 99}]"
get(url) get(url)
urls = extract(".subject a[href*='#{@groupname}']") { |a| a["href"].sub("/d/topic/", "/forum/?_escaped_fragment_=topic/") } topic_urls = extract(".subject a[href*='#{@groupname}']") { |a| a["href"].sub("/d/topic/", "/forum/?_escaped_fragment_=topic/") }
break if urls.size == 0 break if topic_urls.size == 0
urls.each { |url| crawl_topic(url) } topic_urls.each { |topic_url| crawl_topic(topic_url) }
end end
end end
@ -94,7 +94,7 @@ def crawl_topic(url)
a["href"].sub("/d/msg/", "/forum/message/raw?msg="), a["href"].sub("/d/msg/", "/forum/message/raw?msg="),
a["title"].empty? a["title"].empty?
] ]
end.each { |url, might_be_deleted| crawl_message(url, might_be_deleted) } end.each { |msg_url, might_be_deleted| crawl_message(msg_url, might_be_deleted) }
@scraped_topic_urls << url @scraped_topic_urls << url
rescue rescue