[{"id":"Google_Maps_Local_Business_Scraper","platform":"Lead Generation","name":"Google Maps Local Business Scraper","description":"","project":{"commands":[{"command":"fill","defaultName":"Fill Input","description":"Fills an input element with text.","name":"Fill Input","params":{"clear_before_fill":true,"iframe":"","selector":"css=input[name='q']","timeout":5000,"value":"${search_term}"},"then":[{"command":"press","defaultName":"Press Key","description":"Simulates a key press on an element (e.g., 'Enter').","name":"Press Key","params":{"iframe":"","key":"Enter","selector":"css=input[name='q']","timeout":5000},"then":[]}]},{"command":"wait_for_selector","defaultName":"Wait for Selector","description":"Waits for an element to appear in the DOM.","name":"Wait for Selector","params":{"iframe":"","selector":"css=div[aria-label*='Results for '] div[role='article']","timeout":30000},"then":[]},{"category":"Logic","command":"infinite_scroll","defaultName":"Infinite Scroll","description":"Scrolls until the bottom is reached or a max count is hit. Executes nested blocks first (e.g., extracting data step-by-step).","name":"Infinite Scroll","params":{"do":[],"max_scrolls":0,"scroll_delay":1500,"selector":"css=div[aria-label*='Results for ']"}},{"command":"extract_data","defaultName":"Extract Data","description":"Runs a defined data extraction job and saves the results.","name":"Extract Data","params":{"append":true,"dataset":"Map_Search_URLs","filter_condition":"","output_filename":"","output_variable":"","parent_data_from":""},"then":[]},{"command":"for_each","defaultName":"For Each Loop","description":"Loops through items from a previously extracted dataset or a list variable.","name":"For Each Loop","paramOptions":{"source_type":["dataset","variable"]},"params":{"as":"item","dataset":"Map_Search_URLs","do":[{"command":"goto","defaultName":"Go to URL","description":"Navigates to a specific URL.","name":"Go to URL","params":{"url":"${item.PlaceURL}"},"then":[]},{"command":"wait_for_selector","defaultName":"Wait for Selector","description":"Waits for an element to appear in the DOM.","name":"Wait for Selector","params":{"iframe":"","selector":"css=h1","timeout":30000},"then":[]},{"command":"extract_data","defaultName":"Extract Business Details","description":"Runs a defined data extraction job and saves the results.","name":"Extract Business Details","params":{"append":true,"dataset":"Business_Details","filter_condition":"","output_filename":"","output_variable":"","parent_data_from":""},"then":[]}],"source_type":"dataset","variable":""}}],"datasets":{"Business_Details":{"item_selector":"","parent":null,"properties":[{"download":false,"id":"cat-001","key":"Category","name":"Category","selector":"css=button[jsaction*='category']","transformations":[],"type":"text"},{"download":false,"id":"d312a5f7-3d02-49d1-800d-a701cd11fb80","key":"Title","name":"Title","selector":"css=h1","transformations":[],"type":"text"},{"attribute":"","download":false,"id":"amen-001","key":"Amenities","name":"Amenities","selector":"css=div[role=\"region\"][aria-label*=\"About\"]","transformations":[{"code":"const regex = \/aria-label=\"([^\"]+)\"\/g;\nconst matches = [...value.matchAll(regex)].map(match => match[1]);\nreturn matches .join(\"\\n\");","default_value":null,"id":"59cc6369-4dee-460a-b0cf-3d8a0213b666","type":"script"}],"type":"html"},{"attribute":"aria-label","download":false,"id":"a46ca53e-9f65-46d4-852d-e2e03e76107f","key":"Phone","name":"Phone","selector":"css=[data-item-id*='phone']","transformations":[{"find":"Phone: ","flags":"g","id":"0f5ad252-10fe-4041-a77f-ab2f6a4f1dd1","replace_with":"","type":"replace"}],"type":"attribute"},{"attribute":"href","download":false,"id":"36590c76-6187-4266-a31b-5690b608afc8","key":"Website","name":"Website","selector":"css=[data-item-id*='authority']","transformations":[{"find":"Website: ","flags":"g","id":"846f6efe-ae48-4f02-8617-0213590e98b7","replace_with":"","type":"replace"}],"type":"attribute"},{"attribute":"href","download":false,"id":"fb14b236-7d22-48a9-bbde-70e9e89d9663","key":"Menu","name":"Menu","selector":"css=[data-item-id='menu']","transformations":[],"type":"attribute"},{"download":false,"id":"dd1536ab-f111-4112-9b07-95ed065b37ce","key":"Price_Range","name":"Price_Range","selector":"css=table[aria-label='Price range histogram'] tr:first-child td:first-child","transformations":[],"type":"text"},{"attribute":"aria-label","download":false,"id":"76d614b4-aed9-45b2-beeb-b6d8b38832f9","key":"Rating","name":"Rating","selector":"css=[role='img'][aria-label*='stars' i]","transformations":[{"default_value":"","flags":"i","group":0,"id":"0d2ddacb-a1df-4a2a-ab5a-1449b937c0ca","pattern":"([\\d.,]+)","type":"regex"}],"type":"attribute"},{"download":false,"id":"a636e3c5-5b81-4594-9cf4-cedd8cd82099","key":"Reviews_Count","name":"Reviews_Count","selector":"css=[aria-label*='reviews' i]","transformations":[{"default_value":"","flags":"i","group":0,"id":"b27edb55-52ff-429a-bcda-a6488837b2da","pattern":"([\\d,]+)","type":"regex"}],"type":"text"},{"attribute":"aria-label","download":false,"id":"8c2b3d1e-db0c-4b2f-9c15-43575e46d0f2","key":"Address","name":"Address","selector":"css=[data-item-id*='address' i]","transformations":[{"find":"Address: ","flags":"g","id":"59ef6f95-45f9-4793-9e80-201a5f9a5bbe","replace_with":"","type":"replace"}],"type":"attribute"},{"attribute":"aria-label","download":false,"id":"plus-001","key":"Plus_Code","name":"Plus_Code","selector":"css=[data-item-id='oloc']","transformations":[{"find":"Plus code: ","flags":"g","id":"3bc128c3-9c88-4f90-b525-84a61d03344d","replace_with":"","type":"replace"}],"type":"attribute"}],"required_fields":[]},"Map_Search_URLs":{"item_selector":"css=div[aria-label*='Results for '] div[role='article']","parent":null,"properties":[{"attribute":"href","download":false,"id":"070faf13-c985-46ab-b88f-21bca58b5720","key":"PlaceURL","name":"PlaceURL","selector":"css=a[href*='\/maps\/place\/']","transformations":[],"type":"attribute"}],"required_fields":[]}},"folder":"Lead Generation","name":"Google Maps Local Business Scraper","remote":{"api_key":null,"enabled":false},"settings":{"ai":{"anthropic_api_key":"","local_url":"http:\/\/localhost:8000\/v1","openai_api_key":"","openrouter_api_key":""},"ai_self_correction":false,"block_resources":{"enabled":false,"types":[]},"browserProfileId":null,"get_urls_from_sitemap":{"enabled":false,"exclude":[],"include":[]},"headless":false,"logging":{"filename":null,"level":2},"max_concurrent_workers":1,"navigation_timeout":30000,"respect_robots_txt":false,"urls":["https:\/\/www.google.com\/maps\/?hl=en"],"user_agent":null,"variables":[{"description":"","is_list":false,"is_secure":false,"name":"search_term","required":true,"value":"coffee shops"}]},"triggers":[]}},{"id":"SEO_Rank_Tracker.rtila","platform":"SEO","name":"SEO Rank Tracker","description":"","project":{"commands":[{"command":"for_each","defaultName":"For Each Loop","description":"Loops through items from a previously extracted dataset or a list variable.","name":"For Each Loop","paramOptions":{"source_type":["dataset","variable"]},"params":{"as":"keyword","dataset":"","do":[{"command":"goto","defaultName":"Go to URL","description":"Navigates to a specific URL.","name":"Go to URL","params":{"url":"https:\/\/google.com\/search?q=${keyword}&udm=14"},"then":[]},{"command":"extract_data","defaultName":"Extract Data","description":"Runs a defined data extraction job and saves the results.","name":"Extract Data #2","params":{"append":true,"dataset":"Default","filter_condition":"item.URL.includes(item.Domain)","output_filename":"","output_variable":"","parent_data_from":""},"then":[]}],"source_type":"variable","variable":"keywords"}}],"datasets":{"Default":{"item_selector":"xpath=\/\/div[@id='rso']\/\/a[h3]\/ancestor::div[contains(@class, 'g')]","parent":null,"properties":[{"attribute":"content","download":false,"id":"1255232a-a27e-40b5-a570-ddc0d14cec1d","key":"Domain","name":"Domain","selector":"variable=domain","transformations":[],"type":"attribute"},{"attribute":"content","download":false,"id":"93ca77e0-33ef-4e77-a0cc-7d17aea27bde","key":"Keyword","name":"Keyword","selector":"variable=keyword","transformations":[],"type":"attribute"},{"attribute":"href","download":false,"id":"c9da6478-64c9-48bb-a692-b946827d5dee","key":"URL","name":"URL","selector":"css=#rso a:has(h3)","transformations":[],"type":"attribute"},{"download":false,"id":"efd6f301-71d8-41cd-8aaa-b899ba635a45","key":"Rank","name":"Rank","selector":"","transformations":[],"type":"index"}],"required_fields":[]}},"folder":"SEO","name":"SEO Rank Tracker","remote":{"api_key":null,"enabled":false},"settings":{"ai":{"anthropic_api_key":"","local_url":"http:\/\/localhost:8000\/v1","openai_api_key":"","openrouter_api_key":""},"ai_self_correction":true,"block_resources":{"enabled":false,"types":[]},"browserProfileId":null,"get_urls_from_sitemap":{"enabled":false,"exclude":[],"include":[]},"headless":false,"logging":{"filename":null,"level":2},"max_concurrent_workers":1,"navigation_timeout":30000,"respect_robots_txt":false,"urls":["https:\/\/www.google.com\/"],"user_agent":null,"variables":[{"description":"","is_list":false,"is_secure":false,"name":"domain","required":true,"value":"amazon.com"},{"description":"","is_list":true,"is_secure":false,"name":"keywords","required":true,"value":"stainless steel water bottle\nergonomic office chair\nnoise cancelling headphones"}]},"triggers":[]}},{"id":"bdjobs_html_search.rtila","platform":"BDJobs","name":"bdjobs_html_search","description":"Scrapes BDJobs using standard HTML DOM parsing and pagination.","project":{"name":"bdjobs_html_search","version":"1.0.0","folder":"BDJobs","description":"Scrapes BDJobs using standard HTML DOM parsing and pagination.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"search_term","value":"software engineer","required":true,"description":"Job title or keyword"},{"name":"pages_to_scrape","value":5,"required":true,"description":"Number of pages to scrape"}]},"commands":[{"command":"set_variable","params":{"name":"current_page","value":1}},{"command":"string_operation","params":{"value":"${search_term}","operation":"url_encode","output_variable":"encoded_search"}},{"command":"while","params":{"condition":{"type":"script","expression":"variables.current_page <= variables.pages_to_scrape"},"do":[{"command":"goto","params":{"url":"https:\/\/jobs.bdjobs.com\/jobsearch.asp?txtsearch=${encoded_search}&pg=${current_page}"}},{"command":"extract_data","params":{"dataset":"bdjobs_listings"}},{"command":"math_operation","params":{"left_operand":"${current_page}","right_operand":1,"operator":"+","output_variable":"current_page"}},{"command":"wait","params":{"timeout":2500}}]}}],"datasets":{"bdjobs_listings":{"item_selector":"css=.job-item, css=.sout-jobs-wrapper, css=.norm-jobs-wrapper","deduplicate_by":["job_url"],"properties":[{"name":"title","selector":"css=.job-title-text","type":"text"},{"name":"company","selector":"css=.comp-name-text","type":"text"},{"name":"location","selector":"css=.locon-text-d","type":"text"},{"name":"job_url","selector":"css=a[href*='jobdetail']","type":"attribute","attribute":"href"}]}}}},{"id":"glassdoor_graphql_search.rtila","platform":"Glassdoor","name":"glassdoor_graphql_search","description":"Scrapes Glassdoor jobs using GraphQL, handling dynamic CSRF tokens.","project":{"name":"glassdoor_graphql_search","version":"1.0.0","folder":"Glassdoor","description":"Scrapes Glassdoor jobs using GraphQL, handling dynamic CSRF tokens.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"search_term","value":"software engineer","required":true,"description":"Job title or keyword"},{"name":"pages_to_scrape","value":3,"required":true,"description":"Number of pages to scrape (30 jobs per page)"}]},"commands":[{"command":"goto","params":{"url":"https:\/\/www.glassdoor.com\/Job\/computer-science-jobs.htm"}},{"command":"run_script","params":{"script":"const content = await page.content();\nconst match = content.match(\/\"token\":\\s*\"([^\"]+)\"\/);\nvariables.gd_csrf_token = match ? match[1] : \"Ft6oHEWlRZrxDww95Cpazw:0pGUrkb2y3TyOpAIqF2vbPmUXoXVkD3oEGDVkvfeCerceQ5-n8mBg3BovySUIjmCPHCaW0H2nQVdqzbtsYqf4Q:wcqRqeegRUa9MVLJGyujVXB7vWFPjdaS1CtrrzJq-ok\";"}},{"command":"set_variable","params":{"name":"cursor","value":""}},{"command":"set_variable","params":{"name":"current_page","value":1}},{"command":"set_variable","params":{"name":"has_more","value":true}},{"command":"while","params":{"condition":{"type":"script","expression":"variables.current_page <= variables.pages_to_scrape && variables.has_more === true"},"do":[{"command":"run_script","params":{"script":"const payload = [{\n  operationName: \"JobSearchResultsQuery\",\n  variables: {\n    excludeJobListingIds: [],\n    filterParams: [],\n    keyword: variables.search_term,\n    numJobsToShow: 30,\n    locationType: \"STATE\",\n    locationId: 11047,\n    parameterUrlInput: \"IL.0,12_ISTATE11047\",\n    pageNumber: variables.current_page,\n    pageCursor: variables.cursor || null,\n    sort: \"date\"\n  },\n  query: \"query JobSearchResultsQuery($excludeJobListingIds: [Long!], $keyword: String, $locationId: Int, $locationType: LocationTypeEnum, $numJobsToShow: Int!, $pageCursor: String, $pageNumber: Int, $filterParams: [FilterParams], $originalPageUrl: String, $seoFriendlyUrlInput: String, $parameterUrlInput: String, $seoUrl: Boolean) { jobListings(contextHolder: {searchParams: {excludeJobListingIds: $excludeJobListingIds, keyword: $keyword, locationId: $locationId, locationType: $locationType, numPerPage: $numJobsToShow, pageCursor: $pageCursor, pageNumber: $pageNumber, filterParams: $filterParams, originalPageUrl: $originalPageUrl, seoFriendlyUrlInput: $seoFriendlyUrlInput, parameterUrlInput: $parameterUrlInput, seoUrl: $seoUrl, searchType: SR}}) { paginationCursors { cursor pageNumber } jobListings { jobview { header { employerNameFromSearch locationName } job { listingId jobTitleText } } } } }\"\n}];\nvariables.glassdoor_payload = JSON.stringify(payload);"}},{"command":"http_request","params":{"url":"https:\/\/www.glassdoor.com\/graph","method":"POST","use_browser_context":true,"headers":{"gd-csrf-token":"${gd_csrf_token}","apollographql-client-name":"job-search-next","content-type":"application\/json"},"body":"${glassdoor_payload}","output_variable":"glassdoor_response"}},{"command":"run_script","params":{"script":"const res = variables.glassdoor_response;\nif (!res || !res.json || !res.json[0] || !res.json[0].data) {\n  variables.has_more = false;\n  return;\n}\nconst data = res.json[0].data.jobListings;\nconst jobs = data.jobListings.map(j => {\n  const jv = j.jobview;\n  return {\n    id: jv.job.listingId,\n    title: jv.job.jobTitleText,\n    company: jv.header.employerNameFromSearch,\n    location: jv.header.locationName,\n    url: `https:\/\/www.glassdoor.com\/job-listing\/j?jl=${jv.job.listingId}`\n  };\n});\nif (jobs.length > 0) {\n  await helpers.saveData('glassdoor_jobs', jobs);\n}\nconst cursors = data.paginationCursors || [];\nconst nextCursorObj = cursors.find(c => c.pageNumber === variables.current_page + 1);\nvariables.cursor = nextCursorObj ? nextCursorObj.cursor : null;\nvariables.has_more = !!variables.cursor;"}},{"command":"math_operation","params":{"left_operand":"${current_page}","right_operand":1,"operator":"+","output_variable":"current_page"}},{"command":"wait","params":{"timeout":3000}}]}}],"datasets":{"glassdoor_jobs":{"item_selector":"css=html","deduplicate_by":["id"],"properties":[]}}}},{"id":"indeed_graphql_search.rtila","platform":"Indeed","name":"indeed_graphql_search","description":"Scrapes Indeed jobs using their private GraphQL API.","project":{"name":"indeed_graphql_search","version":"1.0.0","folder":"Indeed","description":"Scrapes Indeed jobs using their private GraphQL API.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"search_term","value":"software engineer","required":true,"description":"Job title or keyword"},{"name":"location","value":"New York, NY","required":true,"description":"Job location"},{"name":"pages_to_scrape","value":3,"required":true,"description":"Number of pages to scrape (100 jobs per page)"}]},"commands":[{"command":"set_variable","params":{"name":"cursor","value":""}},{"command":"set_variable","params":{"name":"current_page","value":1}},{"command":"set_variable","params":{"name":"has_more","value":true}},{"command":"while","params":{"condition":{"type":"script","expression":"variables.current_page <= variables.pages_to_scrape && variables.has_more === true"},"do":[{"command":"run_script","params":{"script":"const query = `query GetJobData { jobSearch(what: \"${variables.search_term}\", location: {where: \"${variables.location}\", radius: 50, radiusUnit: MILES}, limit: 100, cursor: \"${variables.cursor}\", sort: RELEVANCE) { pageInfo { nextCursor } results { job { key title datePublished description { html } location { city admin1Code countryCode } employer { name } } } } }`;\nvariables.graphql_payload = JSON.stringify({ query });"}},{"command":"http_request","params":{"url":"https:\/\/apis.indeed.com\/graphql","method":"POST","use_browser_context":true,"headers":{"indeed-api-key":"161092c2017b5bbab13edb12461a62d5a833871e7cad6d9d475304573de67ac8","indeed-co":"US","content-type":"application\/json"},"body":"${graphql_payload}","output_variable":"indeed_response"}},{"command":"run_script","params":{"script":"const res = variables.indeed_response;\nif (!res || !res.json || !res.json.data || !res.json.data.jobSearch) {\n  variables.has_more = false;\n  return;\n}\nconst data = res.json.data.jobSearch;\nconst jobs = data.results.map(r => {\n  const j = r.job;\n  return {\n    id: j.key,\n    title: j.title,\n    company: j.employer ? j.employer.name : null,\n    location: j.location ? `${j.location.city || ''}, ${j.location.admin1Code || ''}` : null,\n    date_published: j.datePublished,\n    url: `https:\/\/www.indeed.com\/viewjob?jk=${j.key}`\n  };\n});\nif (jobs.length > 0) {\n  await helpers.saveData('indeed_jobs', jobs);\n}\nvariables.cursor = data.pageInfo ? data.pageInfo.nextCursor : null;\nvariables.has_more = !!variables.cursor;"}},{"command":"math_operation","params":{"left_operand":"${current_page}","right_operand":1,"operator":"+","output_variable":"current_page"}},{"command":"wait","params":{"timeout":3000}}]}}],"datasets":{"indeed_jobs":{"item_selector":"css=html","deduplicate_by":["id"],"properties":[]}}}},{"id":"instagram_hashtag_search.rtila","platform":"Instagram","name":"instagram_hashtag_search","description":"Scrapes Instagram explore timeline matching targeted hashtags utilizing browser contextual extraction routines simulating human interactions.","project":{"name":"instagram_hashtag_search","version":"1.0.0","folder":"Instagram","description":"Scrapes Instagram explore timeline matching targeted hashtags utilizing browser contextual extraction routines simulating human interactions.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"captchaEnabled":true,"max_concurrent_workers":1,"navigation_timeout":60000,"skip_initial_navigation":false,"variables":[{"name":"target_hashtag","value":"spacex","required":true,"description":"Instagram hashtag keyword to scrape (without # prefix)"},{"name":"max_pagination_scrolls","value":10,"required":false,"description":"Max scrolls to trigger lazy loading of deeper timeline search outputs"}]},"commands":[{"command":"goto","params":{"url":"https:\/\/www.instagram.com\/explore\/tags\/${target_hashtag}\/"}},{"command":"wait","params":{"timeout":4000}},{"command":"infinite_scroll","params":{"max_scrolls":"${max_pagination_scrolls}","scroll_delay":3500,"do":[{"command":"wait","params":{"timeout":1500}},{"command":"extract_data","params":{"dataset":"hashtag_artifacts","output_filename":"instagram_${target_hashtag}_results.json","append":true}}]}}],"datasets":{"hashtag_artifacts":{"item_selector":"css=a[href*='\/p\/'], css=a[href*='\/reel\/']","deduplicate_by":["post_url"],"properties":[{"name":"post_url","selector":"css=self","type":"attribute","attribute":"href"},{"name":"shortcode","selector":"css=self","type":"attribute","attribute":"href","transformations":[{"type":"regex","pattern":"\\\/(?:p|reel)\\\/([a-zA-Z0-9_-]+)\\\/","group":1}]},{"name":"media_file","selector":"css=img","type":"attribute","attribute":"src","download":true},{"name":"caption_text","selector":"css=img","type":"attribute","attribute":"alt"}]}}}},{"id":"instagram_profile_posts.rtila","platform":"Instagram","name":"instagram_profile_posts","description":"Scrapes an Instagram profile's entire visible post footprint natively avoiding intense cryptographic API session challenges utilizing anti-bot DOM emulation.","project":{"name":"instagram_profile_posts","version":"1.0.0","folder":"Instagram","description":"Scrapes an Instagram profile's entire visible post footprint natively avoiding intense cryptographic API session challenges utilizing anti-bot DOM emulation.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"captchaEnabled":true,"max_concurrent_workers":1,"navigation_timeout":60000,"skip_initial_navigation":false,"variables":[{"name":"target_profile","value":"spacex","required":true,"description":"Instagram profile username to scrape (without @ prefix)"},{"name":"max_pagination_scrolls","value":5,"required":false,"description":"Max scrolls to trigger lazy loading of historic timeline posts"}]},"commands":[{"command":"goto","params":{"url":"https:\/\/www.instagram.com\/${target_profile}\/"}},{"command":"wait","params":{"timeout":4000}},{"command":"infinite_scroll","params":{"max_scrolls":"${max_pagination_scrolls}","scroll_delay":3500,"do":[{"command":"wait","params":{"timeout":1500}},{"command":"extract_data","params":{"dataset":"instagram_posts","output_filename":"instagram_${target_profile}_posts.json","append":true}}]}}],"datasets":{"instagram_posts":{"item_selector":"css=a[href*='\/p\/'], css=a[href*='\/reel\/']","deduplicate_by":["shortcode"],"properties":[{"name":"post_url","selector":"css=self","type":"attribute","attribute":"href"},{"name":"shortcode","selector":"css=self","type":"attribute","attribute":"href","transformations":[{"type":"regex","pattern":"\\\/(?:p|reel)\\\/([a-zA-Z0-9_-]+)\\\/","group":1}]},{"name":"media_file","selector":"css=img","type":"attribute","attribute":"src","download":true},{"name":"caption_text","selector":"css=img","type":"attribute","attribute":"alt"},{"name":"is_video","selector":"css=svg[aria-label='Video'], css=span.video-icon, css=span.CustomCssForVideo","type":"count"}]}}}},{"id":"linkedin_company.rtila","platform":"LinkedIn","name":"linkedin_company","description":"Scrapes LinkedIn Company details.","project":{"name":"linkedin_company","version":"1.0.0","folder":"LinkedIn","description":"Scrapes LinkedIn Company details.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"company_id","value":"microsoft","required":true,"description":"LinkedIn Company ID"},{"name":"li_at_cookie","value":"","required":true,"description":"LinkedIn authentication cookie (li_at)"}]},"commands":[{"command":"set_cookie","params":{"name":"li_at","value":"${li_at_cookie}","domain":".linkedin.com"}},{"command":"goto","params":{"url":"https:\/\/www.linkedin.com\/company\/${company_id}\/"}},{"command":"wait_for_selector","params":{"selector":"h1","timeout":15000}},{"command":"extract_data","params":{"dataset":"company_details","output_filename":"linkedin_${company_id}_company.json"}}],"datasets":{"company_details":{"item_selector":"css=main","deduplicate_by":["name"],"properties":[{"name":"name","selector":"css=h1","type":"text"},{"name":"about","selector":"css=section p","type":"text"},{"name":"website","selector":"css=a[href*='http']","type":"attribute","attribute":"href"},{"name":"info_items","selector":"css=.org-top-card-summary-info-list__info-item","type":"list","properties":[{"name":"text","selector":"css=self","type":"text"}]}]}}}},{"id":"linkedin_company_posts.rtila","platform":"LinkedIn","name":"linkedin_company_posts","description":"Scrapes LinkedIn Company posts.","project":{"name":"linkedin_company_posts","version":"1.0.0","folder":"LinkedIn","description":"Scrapes LinkedIn Company posts.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"company_id","value":"microsoft","required":true,"description":"LinkedIn Company ID"},{"name":"li_at_cookie","value":"","required":true,"description":"LinkedIn authentication cookie (li_at)"},{"name":"max_scrolls","value":"10","required":false,"description":"Number of times to scroll for more posts"}]},"commands":[{"command":"set_cookie","params":{"name":"li_at","value":"${li_at_cookie}","domain":".linkedin.com"}},{"command":"goto","params":{"url":"https:\/\/www.linkedin.com\/company\/${company_id}\/posts\/"}},{"command":"wait_for_selector","params":{"selector":"[data-urn^='urn:li:activity:']","timeout":15000}},{"command":"infinite_scroll","params":{"max_scrolls":"${max_scrolls}","scroll_delay":2000}},{"command":"extract_data","params":{"dataset":"company_posts","output_filename":"linkedin_${company_id}_posts.json"}}],"datasets":{"company_posts":{"item_selector":"css=[data-urn^='urn:li:activity:']","deduplicate_by":["urn"],"properties":[{"name":"urn","selector":"css=self","type":"attribute","attribute":"data-urn"},{"name":"text","selector":"css=.feed-shared-update-v2__description, css=.break-words","type":"text"},{"name":"posted_date","selector":"css=[class*='actor__sub-description']","type":"text"},{"name":"reactions","selector":"css=[class*='social-details-social-counts__reactions'], css=button[aria-label*='reaction']","type":"text"},{"name":"comments","selector":"css=button[aria-label*='comment']","type":"text"},{"name":"reposts","selector":"css=button[aria-label*='repost']","type":"text"},{"name":"images","selector":"css=img[src*='media']","type":"list","properties":[{"name":"url","selector":"css=self","type":"attribute","attribute":"src"}]}]}}}},{"id":"linkedin_job.rtila","platform":"LinkedIn","name":"linkedin_job","description":"Scrapes a single LinkedIn Job posting.","project":{"name":"linkedin_job","version":"1.0.0","folder":"LinkedIn","description":"Scrapes a single LinkedIn Job posting.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"job_id","value":"1234567890","required":true,"description":"LinkedIn Job ID"}]},"commands":[{"command":"goto","params":{"url":"https:\/\/www.linkedin.com\/jobs\/view\/${job_id}\/"}},{"command":"wait_for_selector","params":{"selector":"h1","timeout":15000}},{"command":"extract_data","params":{"dataset":"job_details","output_filename":"linkedin_job_${job_id}.json"}}],"datasets":{"job_details":{"item_selector":"css=main","deduplicate_by":["title"],"properties":[{"name":"title","selector":"css=h1","type":"text"},{"name":"company","selector":"css=a[href*='\/company\/']","type":"text"},{"name":"company_url","selector":"css=a[href*='\/company\/']","type":"attribute","attribute":"href"},{"name":"location","selector":"css=.job-details-jobs-unified-top-card__primary-description-container","type":"text"},{"name":"description","selector":"css=article","type":"text"}]}}}},{"id":"linkedin_job_search.rtila","platform":"LinkedIn","name":"linkedin_job_search","description":"Scrapes LinkedIn Job Search results.","project":{"name":"linkedin_job_search","version":"1.0.0","folder":"LinkedIn","description":"Scrapes LinkedIn Job Search results.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"keywords","value":"software engineer","required":true,"description":"Job search keywords"},{"name":"location","value":"San Francisco, CA","required":true,"description":"Job location"},{"name":"max_scrolls","value":"5","required":false,"description":"Number of times to scroll for more jobs"}]},"commands":[{"command":"goto","params":{"url":"https:\/\/www.linkedin.com\/jobs\/search\/?keywords=${keywords}&location=${location}"}},{"command":"wait_for_selector","params":{"selector":"a[href*='\/jobs\/view\/']","timeout":15000}},{"command":"infinite_scroll","params":{"max_scrolls":"${max_scrolls}","scroll_delay":1500}},{"command":"extract_data","params":{"dataset":"job_results","output_filename":"linkedin_jobs_search.json"}}],"datasets":{"job_results":{"item_selector":"css=a[href*='\/jobs\/view\/']","deduplicate_by":["url"],"properties":[{"name":"url","selector":"css=self","type":"attribute","attribute":"href"},{"name":"title","selector":"css=self","type":"text"}]}}}},{"id":"linkedin_job_search_2.rtila","platform":"LinkedIn","name":"linkedin_job_search","description":"Scrapes LinkedIn jobs using the backend HTML snippet API.","project":{"name":"linkedin_job_search","version":"1.0.0","folder":"LinkedIn","description":"Scrapes LinkedIn jobs using the backend HTML snippet API.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"search_term","value":"software engineer","required":true,"description":"Job title or keyword"},{"name":"location","value":"United States","required":true,"description":"Job location"},{"name":"pages_to_scrape","value":5,"required":true,"description":"Number of pages to scrape (25 jobs per page)"}]},"commands":[{"command":"set_variable","params":{"name":"current_start","value":0}},{"command":"set_variable","params":{"name":"current_page","value":1}},{"command":"set_variable","params":{"name":"has_more","value":true}},{"command":"string_operation","params":{"value":"${search_term}","operation":"url_encode","output_variable":"encoded_search"}},{"command":"string_operation","params":{"value":"${location}","operation":"url_encode","output_variable":"encoded_location"}},{"command":"while","params":{"condition":{"type":"script","expression":"variables.current_page <= variables.pages_to_scrape && variables.has_more === true"},"do":[{"command":"http_request","params":{"url":"https:\/\/www.linkedin.com\/jobs-guest\/jobs\/api\/seeMoreJobPostings\/search?keywords=${encoded_search}&location=${encoded_location}&start=${current_start}","method":"GET","use_browser_context":true,"output_variable":"linkedin_response"}},{"command":"run_script","params":{"script":"const html = variables.linkedin_response.body;\nif (!html || html.trim() === '') {\n  variables.has_more = false;\n  return;\n}\nawait page.setContent(html);\nvariables.has_more = true;"}},{"command":"extract_data","params":{"dataset":"linkedin_jobs"}},{"command":"math_operation","params":{"left_operand":"${current_start}","right_operand":25,"operator":"+","output_variable":"current_start"}},{"command":"math_operation","params":{"left_operand":"${current_page}","right_operand":1,"operator":"+","output_variable":"current_page"}},{"command":"wait","params":{"timeout":2500}}]}}],"datasets":{"linkedin_jobs":{"item_selector":"css=li, css=.base-search-card","deduplicate_by":["job_url"],"properties":[{"name":"title","selector":"css=.base-search-card__title, css=.sr-only","type":"text"},{"name":"company","selector":"css=.base-search-card__subtitle","type":"text"},{"name":"location","selector":"css=.job-search-card__location","type":"text"},{"name":"job_url","selector":"css=.base-card__full-link","type":"attribute","attribute":"href"},{"name":"date_posted","selector":"css=time","type":"attribute","attribute":"datetime"}]}}}},{"id":"naukri_api_search.rtila","platform":"Naukri","name":"naukri_api_search","description":"Scrapes Naukri jobs using their REST API.","project":{"name":"naukri_api_search","version":"1.0.0","folder":"Naukri","description":"Scrapes Naukri jobs using their REST API.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"search_term","value":"software engineer","required":true,"description":"Job title or keyword"},{"name":"pages_to_scrape","value":5,"required":true,"description":"Number of pages to scrape"}]},"commands":[{"command":"set_variable","params":{"name":"current_page","value":1}},{"command":"set_variable","params":{"name":"has_more","value":true}},{"command":"string_operation","params":{"value":"${search_term}","operation":"url_encode","output_variable":"encoded_search"}},{"command":"while","params":{"condition":{"type":"script","expression":"variables.current_page <= variables.pages_to_scrape && variables.has_more === true"},"do":[{"command":"http_request","params":{"url":"https:\/\/www.naukri.com\/jobapi\/v3\/search?noOfResults=20&urlType=search_by_keyword&searchType=adv&keyword=${encoded_search}&pageNo=${current_page}&k=${encoded_search}","method":"GET","use_browser_context":true,"headers":{"appid":"109","systemid":"Naukri"},"output_variable":"naukri_response"}},{"command":"run_script","params":{"script":"const res = variables.naukri_response;\nif (!res || !res.json || !res.json.jobDetails || res.json.jobDetails.length === 0) {\n  variables.has_more = false;\n  return;\n}\nconst jobs = res.json.jobDetails.map(j => ({\n  id: j.jobId,\n  title: j.title,\n  company: j.companyName,\n  url: `https:\/\/www.naukri.com${j.jdURL || '\/job\/' + j.jobId}`\n}));\nawait helpers.saveData('naukri_jobs', jobs);\nvariables.has_more = true;"}},{"command":"math_operation","params":{"left_operand":"${current_page}","right_operand":1,"operator":"+","output_variable":"current_page"}},{"command":"wait","params":{"timeout":2500}}]}}],"datasets":{"naukri_jobs":{"item_selector":"css=html","deduplicate_by":["id"],"properties":[]}}}},{"id":"openalex_paper_recommend.rtila","platform":"OpenAlex","name":"openalex_paper_recommend","description":"Fetches paper recommendations from OpenAlex API based on a search query.","project":{"name":"openalex_paper_recommend","version":"1.0.0","folder":"OpenAlex","description":"Fetches paper recommendations from OpenAlex API based on a search query.","settings":{"respect_robots_txt":false,"humanoidEnabled":false,"variables":[{"name":"search_query","value":"Memory Sparse Attention","required":true,"description":"The paper title or keywords to search for."},{"name":"limit","value":"10","required":true,"description":"Number of results to fetch."}]},"commands":[{"command":"string_operation","params":{"value":"${search_query}","operation":"url_encode"},"output_variable":"encoded_query"},{"command":"http_request","params":{"url":"https:\/\/api.openalex.org\/works?search=${encoded_query}&per_page=${limit}&sort=relevance_score:desc","method":"GET","retries":3,"retry_delay":2000},"output_variable":"api_response"},{"command":"run_script","params":{"script":"const data = variables.api_response.json;\nif (!data || !data.results) return [];\nconst papers = data.results.map(w => ({\n  id: w.id.replace('https:\/\/openalex.org\/', ''),\n  title: w.title || w.display_name,\n  doi: w.doi,\n  publication_year: w.publication_year,\n  cited_by_count: w.cited_by_count,\n  authors: (w.authorships || []).map(a => a.author.display_name).join(', ')\n}));\nawait helpers.saveData('recommended_papers', papers);\nreturn papers;"}}],"datasets":{"recommended_papers":{"item_selector":"css=html","deduplicate_by":["id"],"properties":[]}}}},{"id":"realtor_property_details.rtila","platform":"Realtor","name":"realtor_property_details","description":"Fetches detailed information for a specific Realtor.com property ID using their internal GraphQL API.","project":{"name":"realtor_property_details","version":"1.0.0","folder":"Realtor","description":"Fetches detailed information for a specific Realtor.com property ID using their internal GraphQL API.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"proxyEnabled":true,"navigation_timeout":60000,"variables":[{"name":"property_id","value":"M91125-85613","required":true,"description":"Property ID to fetch"}]},"commands":[{"command":"run_script","params":{"script":"return JSON.stringify({ operationName: 'GetHomeDetails', query: 'query GetHomeDetails($property_id: ID!) { home(property_id: $property_id) { property_id listing_id status list_price list_date description { beds baths_full baths_half sqft lot_sqft year_built text } location { address { line city state_code postal_code } } primary_photo(https: true) { href } advertisers { name email phones { number } office { name } } tax_history { tax year } } }', variables: { property_id: variables.property_id } });"},"output_variable":"details_payload"},{"command":"http_request","params":{"url":"https:\/\/www.realtor.com\/frontdoor\/graphql","method":"POST","headers":{"Content-Type":"application\/json","User-Agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/135.0.0.0 Safari\/537.36","rdc-client-name":"RDC_WEB_SRP_FS_PAGE","rdc-client-version":"3.0.2515"},"body":"${details_payload}","use_browser_context":true},"output_variable":"details_response"},{"command":"run_script","params":{"script":"const res = variables.details_response; if (!res || !res.json || !res.json.data || !res.json.data.home) return; const r = res.json.data.home; const addr = r.location?.address || {}; const desc = r.description || {}; const mapped = { property_id: r.property_id, listing_id: r.listing_id, status: r.status, price: r.list_price, list_date: r.list_date, beds: desc.beds, baths: (desc.baths_full || 0) + ((desc.baths_half || 0) * 0.5), sqft: desc.sqft, lot_sqft: desc.lot_sqft, year_built: desc.year_built, address: addr.line, city: addr.city, state: addr.state_code, zip: addr.postal_code, photo: r.primary_photo?.href, description: desc.text, advertisers: JSON.stringify(r.advertisers), tax_history: JSON.stringify(r.tax_history) }; await helpers.saveData('property_details', [mapped]);"}}],"datasets":{"property_details":{"item_selector":"css=html","deduplicate_by":["property_id"],"properties":[]}}}},{"id":"realtor_property_search.rtila","platform":"Realtor","name":"realtor_property_search","description":"Searches Realtor.com for properties in a given location using their internal GraphQL API.","project":{"name":"realtor_property_search","version":"1.0.0","folder":"Realtor","description":"Searches Realtor.com for properties in a given location using their internal GraphQL API.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"proxyEnabled":true,"navigation_timeout":60000,"variables":[{"name":"location","value":"Dallas, TX","required":true,"description":"Location to search (e.g., 'Dallas, TX', '85281')"},{"name":"status","value":"for_sale","required":false,"description":"Listing status (for_sale, for_rent, sold, pending)"},{"name":"limit","value":10000,"required":false,"description":"Maximum number of results to fetch"}]},"commands":[{"command":"set_variable","params":{"name":"offset","value":0}},{"command":"run_script","params":{"script":"return JSON.stringify({ operationName: 'Search_suggestions', query: 'query Search_suggestions($searchInput: SearchSuggestionsInput!) { search_suggestions(search_input: $searchInput) { geo_results { geo { area_type city state_code postal_code county centroid { lat lon } _id mpr_id } } } }', variables: { searchInput: { search_term: variables.location } } });"},"output_variable":"suggestion_payload"},{"command":"http_request","params":{"url":"https:\/\/www.realtor.com\/frontdoor\/graphql","method":"POST","headers":{"Content-Type":"application\/json","User-Agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/135.0.0.0 Safari\/537.36","rdc-client-name":"RDC_WEB_SRP_FS_PAGE","rdc-client-version":"3.0.2515"},"body":"${suggestion_payload}","use_browser_context":true},"output_variable":"suggestion_response"},{"command":"run_script","params":{"script":"const res = variables.suggestion_response; if (!res || !res.json || !res.json.data || !res.json.data.search_suggestions || !res.json.data.search_suggestions.geo_results || res.json.data.search_suggestions.geo_results.length === 0) { return { has_more: false, error: 'Location not found' }; } const geo = res.json.data.search_suggestions.geo_results[0].geo; return { has_more: true, geo: geo, offset: 0, limit: parseInt(variables.limit) || 10000 };"},"output_variable":"search_context"},{"command":"while","params":{"condition":{"type":"script","expression":"variables.search_context && variables.search_context.has_more"},"do":[{"command":"run_script","params":{"script":"const ctx = variables.search_context; const status = variables.status || 'for_sale'; let query = `query GetHomeSearch($search_location: SearchLocation, $offset: Int) { homeSearch: home_search(query: { status: [${status}], search_location: $search_location }, limit: 200, offset: $offset) { count total results { property_id listing_id status list_price list_date description { beds baths_full baths_half sqft lot_sqft year_built text } location { address { line city state_code postal_code } } primary_photo(https: true) { href } } } }`; let vars = { offset: ctx.offset, search_location: { location: variables.location } }; return JSON.stringify({ operationName: 'GetHomeSearch', query: query, variables: vars });"},"output_variable":"search_payload"},{"command":"http_request","params":{"url":"https:\/\/www.realtor.com\/frontdoor\/graphql","method":"POST","headers":{"Content-Type":"application\/json","User-Agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/135.0.0.0 Safari\/537.36","rdc-client-name":"RDC_WEB_SRP_FS_PAGE","rdc-client-version":"3.0.2515"},"body":"${search_payload}","use_browser_context":true},"output_variable":"search_response"},{"command":"run_script","params":{"script":"const res = variables.search_response; const ctx = variables.search_context; if (!res || !res.json || !res.json.data || !res.json.data.homeSearch) { ctx.has_more = false; return; } const results = res.json.data.homeSearch.results || []; const total = res.json.data.homeSearch.total || 0; if (results.length > 0) { const mapped = results.map(r => { const addr = r.location?.address || {}; const desc = r.description || {}; return { property_id: r.property_id, listing_id: r.listing_id, status: r.status, price: r.list_price, list_date: r.list_date, beds: desc.beds, baths: (desc.baths_full || 0) + ((desc.baths_half || 0) * 0.5), sqft: desc.sqft, lot_sqft: desc.lot_sqft, year_built: desc.year_built, address: addr.line, city: addr.city, state: addr.state_code, zip: addr.postal_code, photo: r.primary_photo?.href, description: desc.text }; }); await helpers.saveData('properties', mapped); } ctx.offset += results.length; if (results.length === 0 || ctx.offset >= total || ctx.offset >= ctx.limit) { ctx.has_more = false; } return ctx;"},"output_variable":"search_context"}]}}],"datasets":{"properties":{"item_selector":"css=html","deduplicate_by":["property_id"],"properties":[]}}}},{"id":"sogou_wechat_search.rtila","platform":"WeChat","name":"sogou_wechat_search","description":"Searches WeChat articles via Sogou.","project":{"name":"sogou_wechat_search","version":"1.0.0","folder":"WeChat","description":"Searches WeChat articles via Sogou.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"captchaEnabled":true,"navigation_timeout":60000,"variables":[{"name":"keyword","value":"AI Agent","required":true,"description":"The keyword to search for."}]},"commands":[{"command":"string_operation","params":{"value":"${keyword}","operation":"url_encode"},"output_variable":"encoded_keyword"},{"command":"goto","params":{"url":"https:\/\/weixin.sogou.com\/weixin?type=2&query=${encoded_keyword}"}},{"command":"wait_for_selector","params":{"selector":"css=.txt-box","timeout":15000}},{"command":"extract_data","params":{"dataset":"articles","output_filename":"wechat_articles.json"}}],"datasets":{"articles":{"item_selector":"css=.txt-box","deduplicate_by":["url"],"properties":[{"name":"title","selector":"css=h3 a","type":"text","transformations":[{"type":"trim"}]},{"name":"url","selector":"css=h3 a","type":"attribute","attribute":"href","transformations":[{"type":"prefix","value":"https:\/\/weixin.sogou.com"}]},{"name":"snippet","selector":"css=p.txt-info","type":"text","transformations":[{"type":"trim"}]},{"name":"account","selector":"css=a.account","type":"text","transformations":[{"type":"trim"}]}]}}}},{"id":"x_nitter_timeline.rtila","platform":"Twitter","name":"x_nitter_timeline","description":"Scrapes a user's timeline from X\/Twitter via Nitter, handling pagination automatically.","project":{"name":"x_nitter_timeline","version":"1.0.0","folder":"Twitter","description":"Scrapes a user's timeline from X\/Twitter via Nitter, handling pagination automatically.","settings":{"respect_robots_txt":false,"humanoidEnabled":true,"navigation_timeout":60000,"variables":[{"name":"username","value":"elonmusk","required":true,"description":"The X\/Twitter username to scrape (without @)."},{"name":"nitter_instance","value":"nitter.tiekoetter.com","required":true,"description":"The Nitter instance domain to use."},{"name":"max_pages","value":"5","required":true,"description":"Maximum number of pages to scrape."}]},"commands":[{"command":"set_variable","params":{"name":"current_page","value":1}},{"command":"goto","params":{"url":"https:\/\/${nitter_instance}\/${username}"}},{"command":"while","params":{"condition":{"type":"script","expression":"variables.current_page <= parseInt(variables.max_pages)"},"do":[{"command":"extract_data","params":{"dataset":"tweets","output_filename":"${username}_tweets.json"}},{"command":"get_attribute","params":{"selector":"css=.show-more a","attribute":"href","timeout":2000},"output_variable":"next_cursor"},{"command":"if","params":{"condition":{"type":"script","expression":"variables.next_cursor !== null && variables.next_cursor !== undefined"},"then":[{"command":"math_operation","params":{"left_operand":"${current_page}","right_operand":1,"operator":"+"},"output_variable":"current_page"},{"command":"goto","params":{"url":"https:\/\/${nitter_instance}${next_cursor}"}},{"command":"wait","params":{"timeout":2000}}],"else":[{"command":"break","params":[]}]}}]}}],"datasets":{"tweets":{"item_selector":"css=.timeline-item","deduplicate_by":["tweet_id"],"properties":[{"name":"tweet_id","selector":"css=a.tweet-link","type":"attribute","attribute":"href","transformations":[{"type":"regex","pattern":"status\/(\\d+)","group":1}]},{"name":"author","selector":"css=a.username","type":"text"},{"name":"author_name","selector":"css=a.fullname","type":"text"},{"name":"time","selector":"css=.tweet-date a","type":"attribute","attribute":"title"},{"name":"text","selector":"css=.tweet-content","type":"text","transformations":[{"type":"trim"}]},{"name":"replies","selector":"css=.tweet-stat:has(.icon-comment)","type":"text","transformations":[{"type":"cast","to":"integer","default_value":0}]},{"name":"retweets","selector":"css=.tweet-stat:has(.icon-retweet)","type":"text","transformations":[{"type":"cast","to":"integer","default_value":0}]},{"name":"likes","selector":"css=.tweet-stat:has(.icon-heart)","type":"text","transformations":[{"type":"cast","to":"integer","default_value":0}]},{"name":"views","selector":"css=.tweet-stat:has(.icon-stats)","type":"text","transformations":[{"type":"cast","to":"integer","default_value":0}]}]}}}}]