diff --git a/web/config.py b/web/config.py index 4700ff0..2aca4e4 100644 --- a/web/config.py +++ b/web/config.py @@ -324,6 +324,12 @@ THREADED_MODE = True SQLALCHEMY_TRACK_MODIFICATIONS = False ########################################################################## +# Number of records to fetch in one batch in query tool when query result +# set is large. +########################################################################## +ON_DEMAND_RECORD_COUNT = 1000 + +########################################################################## # Local config settings ########################################################################## diff --git a/web/pgadmin/feature_tests/pg_datatype_validation_test.py b/web/pgadmin/feature_tests/pg_datatype_validation_test.py index 69b12f3..0ddf4cf 100644 --- a/web/pgadmin/feature_tests/pg_datatype_validation_test.py +++ b/web/pgadmin/feature_tests/pg_datatype_validation_test.py @@ -93,17 +93,18 @@ class PGDataypeFeatureTest(BaseFeatureTest): self.page.fill_codemirror_area_with(query) self.page.find_by_id("btn-flash").click() wait = WebDriverWait(self.page.driver, 5) - wait.until(EC.presence_of_element_located( - (By.XPATH, "//*[@id='0']//*[@id='datagrid']/div[5]/div/div[1]/div[2]/span"))) + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) # For every sample data-type value, check the expected output. cnt = 2 - for val in expected_output: + cells = canvas.find_elements_by_tag_name('span') + # remove first element as it is row number. + cells.pop(0) + for val, cell in zip(expected_output, cells): try: - source_code = self.page.find_by_xpath( - "//*[@id='0']//*[@id='datagrid']/div[5]/div/div[1]/div[" + str(cnt) + "]/span" - ).get_attribute('innerHTML') - + source_code = cell.get_attribute('innerHTML') PGDataypeFeatureTest.check_result( source_code, expected_output[cnt - 2] diff --git a/web/pgadmin/feature_tests/query_tool_test.py b/web/pgadmin/feature_tests/query_tool_test.py new file mode 100644 index 0000000..7c5884c --- /dev/null +++ b/web/pgadmin/feature_tests/query_tool_test.py @@ -0,0 +1,776 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2017, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +from __future__ import print_function +import time +import sys +import config +from selenium.webdriver import ActionChains +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.common.by import By +from regression.python_test_utils import test_utils +from regression.feature_utils.base_feature_test import BaseFeatureTest + + +class QueryToolFeatureTest(BaseFeatureTest): + """ + This feature test will test the different query tool features. + """ + + scenarios = [ + ("Query tool feature test", dict()) + ] + + def before(self): + connection = test_utils.get_db_connection(self.server['db'], + self.server['username'], + self.server['db_password'], + self.server['host'], + self.server['port']) + test_utils.drop_database(connection, "acceptance_test_db") + test_utils.create_database(self.server, "acceptance_test_db") + self.page.wait_for_spinner_to_disappear() + self._connects_to_server() + self._locate_database_tree_node() + self._open_query_tool() + + def runTest(self): + # on demand result set on scrolling. + print("On demand result set on scrolling...", + file=sys.stderr) + self._on_demand_result() + print("OK.\n", + file=sys.stderr) + self._clear_query_tool() + + # on demand result set on grid select all. + print("On demand result set on grid select all...", + file=sys.stderr) + self._on_demand_result_select_all_grid() + print("OK.\n", + file=sys.stderr) + self._clear_query_tool() + + # on demand result set on column select all. + print("On demand result set on column select all...", + file=sys.stderr) + self._on_demand_result_select_all_column() + print("OK.\n", + file=sys.stderr) + self._clear_query_tool() + + # explain query + print("Explain query...", file=sys.stderr) + self._query_tool_explain() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # explain query with verbose + print("Explain query with verbose...", file=sys.stderr) + self._query_tool_explain_verbose() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # explain query with costs + print("Explain query with costs...", file=sys.stderr) + self._query_tool_explain_cost() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # explain analyze query + print("Explain analyze query...", file=sys.stderr) + self._query_tool_explain_analyze() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # explain analyze query with buffers + print("Explain analyze query with buffers...", file=sys.stderr) + self._query_tool_explain_analyze_buffers() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # explain analyze query with timing + print("Explain analyze query with timing...", file=sys.stderr) + self._query_tool_explain_analyze_timing() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # auto commit disabled. + print("Auto commit disabled...", file=sys.stderr) + self._query_tool_auto_commit_disabled() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # auto commit enabled. + print("Auto commit enabled...", file=sys.stderr) + self._query_tool_auto_commit_enabled() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # auto rollback enabled. + print("Auto rollback enabled...", file=sys.stderr) + self._query_tool_auto_rollback_enabled() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + # cancel query. + print("Cancel query...", file=sys.stderr) + self._query_tool_cancel_query() + print("OK.\n", file=sys.stderr) + self._clear_query_tool() + + def after(self): + self.page.remove_server(self.server) + connection = test_utils.get_db_connection(self.server['db'], + self.server['username'], + self.server['db_password'], + self.server['host'], + self.server['port']) + test_utils.drop_database(connection, "acceptance_test_db") + + def _connects_to_server(self): + self.page.find_by_xpath( + "//*[@class='aciTreeText' and .='Servers']").click() + self.page.driver.find_element_by_link_text("Object").click() + ActionChains(self.page.driver) \ + .move_to_element( + self.page.driver.find_element_by_link_text("Create"))\ + .perform() + self.page.find_by_partial_link_text("Server...").click() + + server_config = self.server + self.page.fill_input_by_field_name("name", server_config['name']) + self.page.find_by_partial_link_text("Connection").click() + self.page.fill_input_by_field_name("host", server_config['host']) + self.page.fill_input_by_field_name("port", server_config['port']) + self.page.fill_input_by_field_name( + "username", + server_config['username'] + ) + self.page.fill_input_by_field_name( + "password", + server_config['db_password'] + ) + self.page.find_by_xpath("//button[contains(.,'Save')]").click() + + def _locate_database_tree_node(self): + self.page.toggle_open_tree_item(self.server['name']) + self.page.toggle_open_tree_item('Databases') + self.page.toggle_open_tree_item('acceptance_test_db') + + def _open_query_tool(self): + self.page.driver.find_element_by_link_text("Tools").click() + + tools_menu = self.page.driver.find_element_by_id('mnu_tools') + + # Query Tool is first li + query_tool = tools_menu.find_element_by_tag_name('li') + + t = time.time() + # wait until Query Tool menu becomes enabled. + while time.time() - t < 20: # 20 seconds + # if menu is disabled then it will have + # two classes 'menu-item disabled'. + # And if menu is enabled the it will have + # only one class 'menu-item'. + if 'menu-item' == str(query_tool.get_attribute('class')): + break + time.sleep(0.1) + else: + assert False, "'Tools -> Query Tool' menu did not enable." + + self.page.find_by_partial_link_text("Query Tool").click() + self.page.click_tab('Query-1') + + # wait until wc docker activates query tool iframe. + # time.sleep(3) + self.page.driver.switch_to_frame( + self.page.driver.find_element_by_tag_name("iframe") + ) + + def _clear_query_tool(self): + # clear codemirror. + self.page.find_by_id("btn-edit").click() + # wait for alertify dialog open animation to complete. + time.sleep(1) + + self.page.click_element(self.page.find_by_xpath("//button[contains(.,'Yes')]")) + # wait for alertify dialog close animation to complete. + time.sleep(1) + + def _on_demand_result(self): + ON_DEMAND_CHUNKS = 2 + query = """-- On demand query result on scroll +SELECT generate_series(1, {}) as id""".format( + config.ON_DEMAND_RECORD_COUNT * ON_DEMAND_CHUNKS) + + wait = WebDriverWait(self.page.driver, 10) + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-flash").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas"))) + + # scroll to bottom to fetch next chunk of result set. + self.driver.execute_script( + "$('.slick-viewport').scrollTop($('.grid-canvas').height());" + ) + + # wait for ajax to complete. + time.sleep(1) + + # again scroll to bottom to bring last row of next chunk in + # viewport. + self.driver.execute_script( + "$('.slick-viewport').scrollTop($('.grid-canvas').height());" + ) + + row_id_to_find = config.ON_DEMAND_RECORD_COUNT * ON_DEMAND_CHUNKS + + canvas.find_element_by_xpath( + '//span[text()="{}"]'.format(row_id_to_find) + ) + + def _on_demand_result_select_all_grid(self): + ON_DEMAND_CHUNKS = 3 + query = """-- On demand query result on grid select all +SELECT generate_series(1, {}) as id""".format( + config.ON_DEMAND_RECORD_COUNT * ON_DEMAND_CHUNKS) + + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-flash").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, ".slick-header-column"))).click() + + # wait for until all records are fetched and selected. + time.sleep(1) + # scroll to bottom to bring last row of next chunk in + # viewport. + self.driver.execute_script( + "$('.slick-viewport').scrollTop($('.grid-canvas').height());" + ) + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) + + row_id_to_find = config.ON_DEMAND_RECORD_COUNT * ON_DEMAND_CHUNKS + + canvas.find_element_by_xpath( + '//span[text()="{}"]'.format(row_id_to_find) + ) + + def _on_demand_result_select_all_column(self): + ON_DEMAND_CHUNKS = 3 + query = """-- On demand query result on column select all +SELECT generate_series(1, {}) as id1, 'dummy' as id2""".format( + config.ON_DEMAND_RECORD_COUNT * ON_DEMAND_CHUNKS) + + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-flash").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + # click on first data column to select all column. + wait.until(EC.presence_of_element_located( + (By.XPATH, + "//div[contains(@class, 'slick-header-column') and not(contains(@class, 'slick-header-columns')) and contains(., 'id1')]") + )).click() + + # wait for until all records are fetched and selected. + time.sleep(1) + # scroll to bottom to bring last row of next chunk in + # viewport. + self.driver.execute_script( + "$('.slick-viewport').scrollTop($('.grid-canvas').height());" + ) + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) + + row_id_to_find = config.ON_DEMAND_RECORD_COUNT * ON_DEMAND_CHUNKS + + canvas.find_element_by_xpath( + '//span[text()="{}"]'.format(row_id_to_find) + ) + + def _query_tool_explain(self): + query = """-- Explain query +SELECT generate_series(1, 1000) as id order by id desc""" + + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + self.page.find_by_id("btn-query-dropdown").click() + self.page.find_by_id("btn-explain").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + self.page.click_tab('Data Output') + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) + # Search for Plan word in result + canvas.find_element_by_xpath("//*[contains(string(),'Plan')]") + + def _query_tool_explain_verbose(self): + query = """-- Explain query with verbose +SELECT generate_series(1, 1000) as id order by id desc""" + + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + query_op = self.page.find_by_id("btn-query-dropdown") + query_op.click() + + ActionChains(self.driver).move_to_element( + query_op.find_element_by_xpath( + "//li[contains(.,'Explain Options')]")).perform() + + self.page.find_by_id("btn-explain-verbose").click() + + self.page.find_by_id("btn-explain").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + self.page.click_tab('Data Output') + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) + # Search for 'Output' word in result + canvas.find_element_by_xpath("//*[contains(string(), 'Output')]") + + def _query_tool_explain_cost(self): + query = """-- Explain query with costs +SELECT generate_series(1, 1000) as id order by id desc""" + + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + query_op = self.page.find_by_id("btn-query-dropdown") + query_op.click() + + ActionChains(self.driver).move_to_element( + query_op.find_element_by_xpath( + "//li[contains(.,'Explain Options')]")).perform() + + self.page.find_by_id("btn-explain-costs").click() + + self.page.find_by_id("btn-explain").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + self.page.click_tab('Data Output') + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) + # Search for 'Total Cost word in result + canvas.find_element_by_xpath("//*[contains(string(),'Total Cost')]") + + def _query_tool_explain_analyze(self): + query = """-- Explain analyze query +SELECT generate_series(1, 1000) as id order by id desc""" + + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-query-dropdown").click() + self.page.find_by_id("btn-explain-analyze").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + self.page.click_tab('Data Output') + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) + # Search for Actual Rows word in result + canvas.find_element_by_xpath("//*[contains(string(),'Actual Rows')]") + + def _query_tool_explain_analyze_buffers(self): + query = """-- Explain analyze query with buffers +SELECT generate_series(1, 1000) as id order by id desc""" + + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + query_op = self.page.find_by_id("btn-query-dropdown") + query_op.click() + + ActionChains(self.driver).move_to_element( + query_op.find_element_by_xpath( + "//li[contains(.,'Explain Options')]")).perform() + + self.page.find_by_id("btn-explain-buffers").click() + + self.page.find_by_id("btn-explain-analyze").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + self.page.click_tab('Data Output') + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) + # Search for 'Shared Read Blocks' word in result + canvas.find_element_by_xpath("//*[contains(string(), 'Shared Read Blocks')]") + + def _query_tool_explain_analyze_timing(self): + query = """-- Explain analyze query with timing +SELECT generate_series(1, 1000) as id order by id desc""" + + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + query_op = self.page.find_by_id("btn-query-dropdown") + query_op.click() + + ActionChains(self.driver).move_to_element( + query_op.find_element_by_xpath( + "//li[contains(.,'Explain Options')]")).perform() + + self.page.find_by_id("btn-explain-timing").click() + + self.page.find_by_id("btn-explain-analyze").click() + + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + self.page.click_tab('Data Output') + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas")) + ) + # Search for 'Actual Total Time' word in result + canvas.find_element_by_xpath("//*[contains(string(), 'Actual Total Time')]") + + def _query_tool_auto_commit_disabled(self): + table_name = 'query_tool_auto_commit_disabled_table' + query = """-- 1. Disable auto commit. +-- 2. Create table in public schema. +-- 3. ROLLBACK transaction. +-- 4. Check if table is *NOT* created. +CREATE TABLE public.{}();""".format(table_name) + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-query-dropdown").click() + + auto_commit_btn = self.page.find_by_id("btn-auto-commit") + + auto_commit_check = auto_commit_btn.find_element_by_tag_name("i") + + # if auto commit is enabled then 'i' element will + # have 'auto-commit fa fa-check' classes + # if auto commit is disabled then 'i' element will + # have 'auto-commit fa fa-check visibility-hidden' classes + + if 'auto-commit fa fa-check' == str(auto_commit_check.get_attribute( + 'class')): + auto_commit_btn.click() + + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self.driver.find_element_by_xpath( + '//div[contains(@class, "sql-editor-message") and contains(string(), "CREATE TABLE")]' + ) + + self._clear_query_tool() + query = """-- 1. (Done) Disable auto commit. +-- 2. (Done) Create table in public schema. +-- 3. ROLLBACK transaction. +-- 4. Check if table is *NOT* created. +ROLLBACK;""" + self.page.fill_codemirror_area_with(query) + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self.driver.find_element_by_xpath( + '//div[contains(@class, "sql-editor-message") and contains(string(), "ROLLBACK")]' + ) + + self._clear_query_tool() + query = """-- 1. (Done) Disable auto commit. +-- 2. (Done) Create table in public schema. +-- 3. (Done) ROLLBACK transaction. +-- 4. Check if table is *NOT* created. +SELECT relname FROM pg_class WHERE relkind IN ('r','s','t') and relnamespace = 2200::oid;""" + self.page.fill_codemirror_area_with(query) + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas"))) + + el = canvas.find_elements_by_xpath("//div[contains(text(), '{}')]".format(table_name)) + + assert len(el) == 0, "Table '{}' created with auto commit disabled and without any explicit commit.".format(table_name) + + def _query_tool_auto_commit_enabled(self): + table_name = 'query_tool_auto_commit_enabled_table' + query = """-- 1. END any open transaction. +-- 2. Enable auto commit. +-- 3. Create table in public schema. +-- 4. ROLLBACK transaction +-- 5. Check if table is created event after ROLLBACK. +END;""".format(table_name) + + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self._clear_query_tool() + + query = """-- 1. (Done) END any open transaction if any. +-- 2. Enable auto commit. +-- 3. Create table in public schema. +-- 4. ROLLBACK transaction +-- 5. Check if table is created event after ROLLBACK. +CREATE TABLE public.{}();""".format(table_name) + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-query-dropdown").click() + + auto_commit_btn = self.page.find_by_id("btn-auto-commit") + + auto_commit_check = auto_commit_btn.find_element_by_tag_name("i") + + # if auto commit is enabled then 'i' element will + # have 'auto-commit fa fa-check' classes + # if auto commit is disabled then 'i' element will + # have 'auto-commit fa fa-check visibility-hidden' classes + + if 'auto-commit fa fa-check visibility-hidden' == str(auto_commit_check.get_attribute( + 'class')): + auto_commit_btn.click() + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self.driver.find_element_by_xpath( + '//div[contains(@class, "sql-editor-message") and contains(string(), "CREATE TABLE")]' + ) + + self._clear_query_tool() + query = """-- 1. (Done) END any open transaction if any. +-- 2. (Done) Enable auto commit. +-- 3. (Done) Create table in public schema. +-- 4. ROLLBACK transaction +-- 5. Check if table is created event after ROLLBACK. +ROLLBACK;""" + self.page.fill_codemirror_area_with(query) + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self.driver.find_element_by_xpath( + '//div[contains(@class, "sql-editor-message") and contains(string(), "ROLLBACK")]' + ) + + self._clear_query_tool() + query = """-- 1. (Done) END any open transaction if any. +-- 2. (Done) Enable auto commit. +-- 3. (Done) Create table in public schema. +-- 4. (Done) ROLLBACK transaction +-- 5. Check if table is created event after ROLLBACK. +SELECT relname FROM pg_class WHERE relkind IN ('r','s','t') and relnamespace = 2200::oid;""" + self.page.fill_codemirror_area_with(query) + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas"))) + + el = canvas.find_elements_by_xpath("//div[contains(text(), '{}')]".format(table_name)) + + assert len(el) != 0, "Table '{}' is not created with auto commit enabled.".format(table_name) + + def _query_tool_auto_rollback_enabled(self): + table_name = 'query_tool_auto_rollback_enabled_table' + query = """-- 1. END any open transaction. +-- 2. Enable auto rollback and disable auto commit. +-- 3. Create table in public schema. +-- 4. Generate error in transaction. +-- 5. END transaction. +-- 6. Check if table is *not* created after ending transaction. +END;""".format(table_name) + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self._clear_query_tool() + + query = """-- 1. (Done) END any open transaction. +-- 2. Enable auto rollback and disable auto commit. +-- 3. Create table in public schema. +-- 4. Generate error in transaction. +-- 5. END transaction. +-- 6. Check if table is *not* created after ending transaction. +CREATE TABLE public.{}();""".format(table_name) + wait = WebDriverWait(self.page.driver, 10) + + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-query-dropdown").click() + + auto_rollback_btn = self.page.find_by_id("btn-auto-rollback") + + auto_rollback_check = auto_rollback_btn.find_element_by_tag_name("i") + + # if auto rollback is enabled then 'i' element will + # have 'auto-rollback fa fa-check' classes + # if auto rollback is disabled then 'i' element will + # have 'auto-rollback fa fa-check visibility-hidden' classes + + if 'auto-rollback fa fa-check visibility-hidden' == str(auto_rollback_check.get_attribute( + 'class')): + auto_rollback_btn.click() + + auto_commit_btn = self.page.find_by_id("btn-auto-commit") + + auto_commit_check = auto_commit_btn.find_element_by_tag_name("i") + + # if auto commit is enabled then 'i' element will + # have 'auto-commit fa fa-check' classes + # if auto commit is disabled then 'i' element will + # have 'auto-commit fa fa-check visibility-hidden' classes + + if 'auto-commit fa fa-check' == str(auto_commit_check.get_attribute( + 'class')): + auto_commit_btn.click() + + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self.driver.find_element_by_xpath( + '//div[contains(@class, "sql-editor-message") and contains(string(), "CREATE TABLE")]' + ) + + self._clear_query_tool() + query = """-- 1. (Done) END any open transaction. +-- 2. (Done) Enable auto rollback and disable auto commit. +-- 3. (Done) Create table in public schema. +-- 4. Generate error in transaction. +-- 5. END transaction. +-- 6. Check if table is *not* created after ending transaction. +SELECT 1/0;""".format(table_name) + self.page.fill_codemirror_area_with(query) + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self.driver.find_element_by_xpath( + '//div[contains(@class, "sql-editor-message") and contains(string(), "division by zero")]' + ) + + self._clear_query_tool() + query = """-- 1. (Done) END any open transaction. +-- 2. (Done) Enable auto rollback and disable auto commit. +-- 3. (Done) Create table in public schema. +-- 4. (Done) Generate error in transaction. +-- 5. END transaction. +-- 6. Check if table is *not* created after ending transaction. +END;""".format(table_name) + self.page.fill_codemirror_area_with(query) + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self.driver.find_element_by_xpath( + '//div[contains(@class, "sql-editor-message") and contains(string(), "ROLLBACK")]' + ) + + self._clear_query_tool() + query = """-- 1. (Done) END any open transaction. +-- 2. (Done) Enable auto rollback and disable auto commit. +-- 3. (Done) Create table in public schema. +-- 4. (Done) Generate error in transaction. +-- 5. (Done) END transaction. +-- 6. Check if table is *not* created after ending transaction. +SELECT relname FROM pg_class WHERE relkind IN ('r','s','t') and relnamespace = 2200::oid;""" + self.page.fill_codemirror_area_with(query) + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + + canvas = wait.until(EC.presence_of_element_located( + (By.CSS_SELECTOR, "#datagrid .slick-viewport .grid-canvas"))) + + el = canvas.find_elements_by_xpath("//div[contains(text(), '{}')]".format(table_name)) + + assert len(el) == 0, "Table '{}' created even after ROLLBACK due to sql error.".format(table_name) + + def _query_tool_cancel_query(self): + query = """-- 1. END any open transaction. +-- 2. Enable auto commit and Disable auto rollback. +-- 3. Execute long running query. +-- 4. Cancel long running execution. +END;""" + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-flash").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self._clear_query_tool() + + query = """-- 1. (Done) END any open transaction. +-- 2. Enable auto commit and Disable auto rollback. +-- 3. Execute long running query. +-- 4. Cancel long running query execution. +SELECT 1, pg_sleep(10)""" + self.page.fill_codemirror_area_with(query) + + self.page.find_by_id("btn-query-dropdown").click() + + auto_rollback_btn = self.page.find_by_id("btn-auto-rollback") + + auto_rollback_check = auto_rollback_btn.find_element_by_tag_name("i") + + # if auto rollback is enabled then 'i' element will + # have 'auto-rollback fa fa-check' classes + # if auto rollback is disabled then 'i' element will + # have 'auto-rollback fa fa-check visibility-hidden' classes + + if 'auto-rollback fa fa-check' == str(auto_rollback_check.get_attribute( + 'class')): + auto_rollback_btn.click() + + auto_commit_btn = self.page.find_by_id("btn-auto-commit") + + auto_commit_check = auto_commit_btn.find_element_by_tag_name("i") + + # if auto commit is enabled then 'i' element will + # have 'auto-commit fa fa-check' classes + # if auto commit is disabled then 'i' element will + # have 'auto-commit fa fa-check visibility-hidden' classes + + if 'auto-commit fa fa-check visibility-hidden' == str(auto_commit_check.get_attribute( + 'class')): + auto_commit_btn.click() + + self.page.find_by_id("btn-flash").click() + self.driver.find_element_by_xpath("//*[@id='fetching_data']") + self.page.find_by_id("btn-cancel-query").click() + self.page.wait_for_query_tool_loading_indicator_to_disappear() + self.driver.find_element_by_xpath( + '//div[contains(@class, "sql-editor-message") and contains(string(), "canceling statement due to user request")]' + ) + diff --git a/web/pgadmin/feature_tests/view_data_dml_queries.py b/web/pgadmin/feature_tests/view_data_dml_queries.py index 0f41c43..8ce2d72 100644 --- a/web/pgadmin/feature_tests/view_data_dml_queries.py +++ b/web/pgadmin/feature_tests/view_data_dml_queries.py @@ -236,8 +236,6 @@ CREATE TABLE public.defaults CheckForViewDataTest._get_cell_xpath("r1", "3") ).click() - # for debugging - print(row1_cell2_xpath) self._compare_cell_value(row1_cell2_xpath, "[default]") # reset cell value to previous one self._update_cell(row1_cell2_xpath, ["1", "", "int"]) diff --git a/web/pgadmin/feature_tests/xss_checks_panels_and_query_tool_test.py b/web/pgadmin/feature_tests/xss_checks_panels_and_query_tool_test.py index d59e8ac..fce3475 100644 --- a/web/pgadmin/feature_tests/xss_checks_panels_and_query_tool_test.py +++ b/web/pgadmin/feature_tests/xss_checks_panels_and_query_tool_test.py @@ -10,6 +10,9 @@ from selenium.webdriver import ActionChains from regression.python_test_utils import test_utils from regression.feature_utils.base_feature_test import BaseFeatureTest +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.common.by import By import time class CheckForXssFeatureTest(BaseFeatureTest): @@ -152,11 +155,16 @@ class CheckForXssFeatureTest(BaseFeatureTest): self.page.fill_codemirror_area_with("select ''") time.sleep(1) self.page.find_by_id("btn-flash").click() - time.sleep(2) + wait = WebDriverWait(self.page.driver, 5) - source_code = self.page.find_by_xpath( - "//*[@id='0']//*[@id='datagrid']/div[5]/div/div[1]/div[2]" - ).get_attribute('innerHTML') + result_row = self.page.find_by_xpath( + "//*[contains(@class, 'ui-widget-content') and contains(@style, 'top:0px')]" + ) + + cells = result_row.find_elements_by_tag_name('div') + + # remove first element as it is row number. + source_code = cells[1].get_attribute('innerHTML') self._check_escaped_characters( source_code, diff --git a/web/pgadmin/static/js/selection/column_selector.js b/web/pgadmin/static/js/selection/column_selector.js index c89b3fa..78d125b 100644 --- a/web/pgadmin/static/js/selection/column_selector.js +++ b/web/pgadmin/static/js/selection/column_selector.js @@ -1,22 +1,34 @@ define(['jquery', 'sources/selection/range_selection_helper', 'slickgrid'], function ($, rangeSelectionHelper) { var ColumnSelector = function () { + var onBeforeColumnSelectAll = new Slick.Event(), + onColumnSelectAll = new Slick.Event(); + var init = function (grid) { grid.onHeaderClick.subscribe(function (event, eventArgument) { var column = eventArgument.column; - if (column.selectable !== false) { - + if (column.selectable !== false && !$(event.target).hasClass('slick-resizable-handle')) { + var $checkbox = $(event.currentTarget).find("[data-id='checkbox-" + column.id + "']"); if (!clickedCheckbox(event)) { - var $checkbox = $("[data-id='checkbox-" + column.id + "']"); toggleCheckbox($checkbox); } - updateRanges(grid, column.id); + if ($checkbox.prop("checked")) { + onBeforeColumnSelectAll.notify(eventArgument, event); + } + + if (!(event.isPropagationStopped() || event.isImmediatePropagationStopped())) { + updateRanges(grid, column.id); + } } } ); grid.getSelectionModel().onSelectedRangesChanged .subscribe(handleSelectedRangesChanged.bind(null, grid)); + + onColumnSelectAll.subscribe(function(e, args) { + updateRanges(args.grid, args.column.id); + }); }; var handleSelectedRangesChanged = function (grid, event, ranges) { @@ -85,7 +97,9 @@ define(['jquery', 'sources/selection/range_selection_helper', 'slickgrid'], func $.extend(this, { "init": init, - "getColumnDefinitionsWithCheckboxes": getColumnDefinitionsWithCheckboxes + "getColumnDefinitionsWithCheckboxes": getColumnDefinitionsWithCheckboxes, + "onBeforeColumnSelectAll": onBeforeColumnSelectAll, + "onColumnSelectAll": onColumnSelectAll }); }; return ColumnSelector; diff --git a/web/pgadmin/static/js/selection/copy_data.js b/web/pgadmin/static/js/selection/copy_data.js index 018efea..d4e8e55 100644 --- a/web/pgadmin/static/js/selection/copy_data.js +++ b/web/pgadmin/static/js/selection/copy_data.js @@ -11,13 +11,12 @@ define([ var grid = self.slickgrid; var columnDefinitions = grid.getColumns(); var selectedRanges = grid.getSelectionModel().getSelectedRanges(); - var data = grid.getData(); + var dataView = grid.getData(); var rows = grid.getSelectedRows(); - if (allTheRangesAreFullRows(selectedRanges, columnDefinitions)) { self.copied_rows = rows.map(function (rowIndex) { - return data[rowIndex]; + return grid.getDataItem(rowIndex); }); setPasteRowButtonEnablement(self.can_edit, true); } else { @@ -25,7 +24,7 @@ define([ setPasteRowButtonEnablement(self.can_edit, false); } - var csvText = rangeBoundaryNavigator.rangesToCsv(data, columnDefinitions, selectedRanges); + var csvText = rangeBoundaryNavigator.rangesToCsv(dataView.getItems(), columnDefinitions, selectedRanges); if (csvText) { clipboard.copyTextToClipboard(csvText); } diff --git a/web/pgadmin/static/js/selection/grid_selector.js b/web/pgadmin/static/js/selection/grid_selector.js index 31aee69..a51d6c9 100644 --- a/web/pgadmin/static/js/selection/grid_selector.js +++ b/web/pgadmin/static/js/selection/grid_selector.js @@ -3,14 +3,18 @@ define(['jquery', 'sources/selection/column_selector', 'sources/selection/row_se var Slick = window.Slick; var GridSelector = function (columnDefinitions) { - var rowSelector = new RowSelector(columnDefinitions); - var columnSelector = new ColumnSelector(columnDefinitions); + var rowSelector = new RowSelector(columnDefinitions), + columnSelector = new ColumnSelector(columnDefinitions), + onBeforeGridSelectAll = new Slick.Event(), + onGridSelectAll = new Slick.Event(), + onBeforeGridColumnSelectAll = columnSelector.onBeforeColumnSelectAll, + onGridColumnSelectAll = columnSelector.onColumnSelectAll; var init = function (grid) { this.grid = grid; grid.onHeaderClick.subscribe(function (event, eventArguments) { - if (eventArguments.column.selectAllOnClick) { - toggleSelectAll(grid); + if (eventArguments.column.selectAllOnClick && !$(event.target).hasClass('slick-resizable-handle')) { + toggleSelectAll(grid, event, eventArguments); } }); @@ -18,6 +22,10 @@ define(['jquery', 'sources/selection/column_selector', 'sources/selection/row_se .subscribe(handleSelectedRangesChanged.bind(null, grid)); grid.registerPlugin(rowSelector); grid.registerPlugin(columnSelector); + + onGridSelectAll.subscribe(function(e, args) { + selectAll(args.grid); + }); }; var getColumnDefinitionsWithCheckboxes = function (columnDefinitions) { @@ -31,7 +39,7 @@ define(['jquery', 'sources/selection/column_selector', 'sources/selection/row_se }; function handleSelectedRangesChanged(grid) { - $("[data-id='checkbox-select-all']").prop("checked", isEntireGridSelected(grid)); + $(grid.getContainerNode()).find("[data-id='checkbox-select-all']").prop("checked", isEntireGridSelected(grid)); } function isEntireGridSelected(grid) { @@ -40,11 +48,15 @@ define(['jquery', 'sources/selection/column_selector', 'sources/selection/row_se return selectedRanges.length == 1 && isSameRange(selectedRanges[0], getRangeOfWholeGrid(grid)); } - function toggleSelectAll(grid) { + function toggleSelectAll(grid, event, eventArguments) { if (isEntireGridSelected(grid)) { deselect(grid); } else { - selectAll(grid) + onBeforeGridSelectAll.notify(eventArguments, event); + + if (!(event.isPropagationStopped() || event.isImmediatePropagationStopped())) { + selectAll(grid); + } } } @@ -71,7 +83,11 @@ define(['jquery', 'sources/selection/column_selector', 'sources/selection/row_se $.extend(this, { "init": init, - "getColumnDefinitionsWithCheckboxes": getColumnDefinitionsWithCheckboxes + "getColumnDefinitionsWithCheckboxes": getColumnDefinitionsWithCheckboxes, + "onBeforeGridSelectAll": onBeforeGridSelectAll, + "onGridSelectAll": onGridSelectAll, + "onBeforeGridColumnSelectAll": onBeforeGridColumnSelectAll, + "onGridColumnSelectAll": onGridColumnSelectAll }); }; diff --git a/web/pgadmin/static/js/selection/range_boundary_navigator.js b/web/pgadmin/static/js/selection/range_boundary_navigator.js index a268d24..e644009 100644 --- a/web/pgadmin/static/js/selection/range_boundary_navigator.js +++ b/web/pgadmin/static/js/selection/range_boundary_navigator.js @@ -57,6 +57,7 @@ define(['sources/selection/range_selection_helper'], function (RangeSelectionHel }, rangesToCsv: function (data, columnDefinitions, selectedRanges) { + var rowRangeBounds = selectedRanges.map(function (range) { return [range.fromRow, range.toRow]; }); @@ -71,6 +72,7 @@ define(['sources/selection/range_selection_helper'], function (RangeSelectionHel var csvRows = this.mapOver2DArray(rowRangeBounds, colRangeBounds, this.csvCell.bind(this, data, columnDefinitions), function (rowData) { return rowData.join(','); }); + return csvRows.join('\n'); }, @@ -96,7 +98,7 @@ define(['sources/selection/range_selection_helper'], function (RangeSelectionHel }, csvCell: function (data, columnDefinitions, rowId, colId) { - var val = data[rowId][columnDefinitions[colId].pos]; + var val = data[rowId][columnDefinitions[colId].field]; if (val && _.isObject(val)) { val = "'" + JSON.stringify(val) + "'"; diff --git a/web/pgadmin/static/js/selection/row_selector.js b/web/pgadmin/static/js/selection/row_selector.js index 76a8c1a..7afa821 100644 --- a/web/pgadmin/static/js/selection/row_selector.js +++ b/web/pgadmin/static/js/selection/row_selector.js @@ -15,9 +15,13 @@ define(['jquery', 'sources/selection/range_selection_helper', 'slickgrid'], func if (grid.getColumns()[args.cell].id === 'row-header-column') { if (event.target.type != "checkbox") { var checkbox = $(event.target).find('input[type="checkbox"]'); - toggleCheckbox($(checkbox)); + if(checkbox.length > 0) { + toggleCheckbox($(checkbox)); + updateRanges(grid, args.row); + } + } else { + updateRanges(grid, args.row); } - updateRanges(grid, args.row); } } @@ -70,7 +74,8 @@ define(['jquery', 'sources/selection/range_selection_helper', 'slickgrid'], func formatter: function (rowIndex) { return '' + 'data-cell-type="row-header-checkbox"/>' + + '' + (rowIndex+1) + '' } }); return columnDefinitions; diff --git a/web/pgadmin/static/js/selection/set_staged_rows.js b/web/pgadmin/static/js/selection/set_staged_rows.js index cace728..76a6cf6 100644 --- a/web/pgadmin/static/js/selection/set_staged_rows.js +++ b/web/pgadmin/static/js/selection/set_staged_rows.js @@ -21,53 +21,45 @@ define( $(selector).prop('disabled', false); } - function getRowPrimaryKeyValuesToStage(selectedRows, primaryKeyColumnIndices, gridData) { + function getRowPrimaryKeyValuesToStage(selectedRows, primaryKeys, dataView, client_primary_key) { return _.reduce(selectedRows, function (primaryKeyValuesToStage, dataGridRowIndex) { - var gridRow = gridData[dataGridRowIndex]; + var gridRow = dataView.getItem(dataGridRowIndex); - if (isRowMissingPrimaryKeys(gridRow, primaryKeyColumnIndices)) { + if (isRowMissingPrimaryKeys(gridRow, primaryKeys)) { return primaryKeyValuesToStage; } - - var tempPK = gridRow.__temp_PK; - primaryKeyValuesToStage[tempPK] = getSingleRowPrimaryKeyValueToStage(primaryKeyColumnIndices, gridRow); - + var tempPK = gridRow[client_primary_key]; + primaryKeyValuesToStage[tempPK] = getSingleRowPrimaryKeyValueToStage(primaryKeys, gridRow); return primaryKeyValuesToStage; }, {}); } - function isRowMissingPrimaryKeys(gridRow, primaryKeyColumnIndices) { + function isRowMissingPrimaryKeys(gridRow, primaryKeys) { if (_.isUndefined(gridRow)) { return true; } return !_.isUndefined( - _.find(primaryKeyColumnIndices, function (pkIndex) { - return _.isUndefined(gridRow[pkIndex]); + _.find(primaryKeys , function (pk) { + return _.isUndefined(gridRow[pk]); }) ); } - function getSingleRowPrimaryKeyValueToStage(primaryKeyColumnIndices, gridRow) { + function getSingleRowPrimaryKeyValueToStage(primaryKeys, gridRow) { var rowToStage = {}; - if (primaryKeyColumnIndices.length) { - _.each(_.keys(gridRow), function (columnPos) { - if (_.contains(primaryKeyColumnIndices, Number(columnPos))) - rowToStage[columnPos] = gridRow[columnPos]; + if (primaryKeys && primaryKeys.length) { + _.each(_.keys(gridRow), function (columnNames) { + if (_.contains(primaryKeys, columnNames)) + rowToStage[columnNames] = gridRow[columnNames]; }) } return rowToStage; } function getPrimaryKeysForSelectedRows(self, selectedRows) { - var primaryKeyColumnIndices = _.map(_.keys(self.keys), function (columnName) { - var columnInfo = _.findWhere(self.columns, {name: columnName}); - return columnInfo['pos']; - }); - - var gridData = self.grid.getData(); - var stagedRows = getRowPrimaryKeyValuesToStage(selectedRows, primaryKeyColumnIndices, gridData); - + var dataView = self.grid.getData(); + var stagedRows = getRowPrimaryKeyValuesToStage(selectedRows, _.keys(self.keys), dataView, self.client_primary_key); return stagedRows; } diff --git a/web/pgadmin/static/js/slickgrid/slick.pgadmin.editors.js b/web/pgadmin/static/js/slickgrid/slick.pgadmin.editors.js index af8141f..307e9d3 100644 --- a/web/pgadmin/static/js/slickgrid/slick.pgadmin.editors.js +++ b/web/pgadmin/static/js/slickgrid/slick.pgadmin.editors.js @@ -50,18 +50,18 @@ last_value = (column_type === 'number') ? (_.isEmpty(last_value) || last_value) : last_value; - item[args.column.pos] = state; + item[args.column.field] = state; if (last_value && _.isNull(state) && (_.isUndefined(grid.copied_rows[row]) || _.isUndefined(grid.copied_rows[row][cell])) ) { - item[args.column.pos] = undefined; + item[args.column.field] = undefined; if (grid.copied_rows[row] == undefined) grid.copied_rows[row] = []; grid.copied_rows[row][cell] = 1; } } else { - item[args.column.pos] = state; + item[args.column.field] = state; } } @@ -156,14 +156,14 @@ this.loadValue = function (item) { var col = args.column; - if (_.isUndefined(item[args.column.pos]) && col.has_default_val) { + if (_.isUndefined(item[args.column.field]) && col.has_default_val) { $input.val(defaultValue = ""); } - else if (item[args.column.pos] === "") { + else if (item[args.column.field] === "") { $input.val(defaultValue = "''"); } else { - $input.val(defaultValue = item[args.column.pos]); + $input.val(defaultValue = item[args.column.field]); $input.select(); } }; @@ -308,7 +308,7 @@ }; this.loadValue = function (item) { - var data = defaultValue = item[args.column.pos]; + var data = defaultValue = item[args.column.field]; if (data && typeof data === "object" && !Array.isArray(data)) { data = JSON.stringify(data); } else if (Array.isArray(data)) { @@ -445,7 +445,7 @@ }; this.loadValue = function (item) { - $input.val(defaultValue = item[args.column.pos]); + $input.val(defaultValue = item[args.column.field]); $input.select(); }; @@ -454,7 +454,7 @@ }; this.applyValue = function (item, state) { - item[args.column.pos] = state; + item[args.column.field] = state; }; this.isValueChanged = function () { @@ -533,13 +533,13 @@ }; this.loadValue = function (item) { - defaultValue = item[args.column.pos]; - if (_.isNull(defaultValue)|| _.isUndefined(defaultValue)) { + defaultValue = item[args.column.field]; + if (_.isNull(defaultValue)||_.isUndefined(defaultValue)) { $select.prop('indeterminate', true); $select.data('checked', 2); } else { - defaultValue = !!item[args.column.pos]; + defaultValue = !!item[args.column.field]; if (defaultValue) { $select.prop('checked', true); $select.data('checked', 0); @@ -558,7 +558,7 @@ }; this.applyValue = function (item, state) { - item[args.column.pos] = state; + item[args.column.field] = state; }; this.isValueChanged = function () { @@ -667,7 +667,7 @@ }; this.loadValue = function (item) { - var data = defaultValue = item[args.column.pos]; + var data = defaultValue = item[args.column.field]; if (typeof data === "object" && !Array.isArray(data)) { data = JSON.stringify(data); } else if (Array.isArray(data)) { @@ -690,7 +690,7 @@ }; this.applyValue = function (item, state) { - item[args.column.pos] = state; + item[args.column.field] = state; }; this.isValueChanged = function () { @@ -744,7 +744,7 @@ }; this.loadValue = function (item) { - var value = item[args.column.pos]; + var value = item[args.column.field]; // Check if value is null or undefined if (value === undefined && typeof value === "undefined") { @@ -877,7 +877,7 @@ }; this.loadValue = function (item) { - defaultValue = item[args.column.pos]; + defaultValue = item[args.column.field]; $input.val(defaultValue); $input[0].defaultValue = defaultValue; $input.select(); diff --git a/web/pgadmin/templates/base.html b/web/pgadmin/templates/base.html index 9cbe2b9..8aa2816 100755 --- a/web/pgadmin/templates/base.html +++ b/web/pgadmin/templates/base.html @@ -145,6 +145,12 @@ ], "exports": 'Slick.Grid' }, + "slickgrid/slick.dataview": { + "deps": [ + "slickgrid" + ], + "exports": 'Slick.Data.DataView' + }, "flotr2": { deps: ['bean'], exports: function(bean) { diff --git a/web/pgadmin/tools/sqleditor/__init__.py b/web/pgadmin/tools/sqleditor/__init__.py index 828cb99..e749f4f 100644 --- a/web/pgadmin/tools/sqleditor/__init__.py +++ b/web/pgadmin/tools/sqleditor/__init__.py @@ -27,7 +27,7 @@ from pgadmin.utils.sqlautocomplete.autocomplete import SQLAutoComplete from pgadmin.misc.file_manager import Filemanager -from config import PG_DEFAULT_DRIVER +from config import PG_DEFAULT_DRIVER, ON_DEMAND_RECORD_COUNT MODULE_NAME = 'sqleditor' @@ -229,13 +229,32 @@ def start_view_data(trans_id): # Check the transaction and connection status status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id) + + # get the default connection as current connection which is attached to + # trans id holds the cursor which has query result so we cannot use that + # connection to execute another query otherwise we'll lose query result. + + manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(trans_obj.sid) + default_conn = manager.connection(did=trans_obj.did) + + # Connect to the Server if not connected. + if not default_conn.connected(): + status, msg = default_conn.connect() + if not status: + return make_json_response( + data={'status': status, 'result': u"{}".format(msg)} + ) + if status and conn is not None \ and trans_obj is not None and session_obj is not None: try: + # set fetched row count to 0 as we are executing query again. + trans_obj.update_fetched_row_cnt(0) + session_obj['command_obj'] = pickle.dumps(trans_obj, -1) # Fetch the sql and primary_keys from the object sql = trans_obj.get_sql() - pk_names, primary_keys = trans_obj.get_primary_keys() + pk_names, primary_keys = trans_obj.get_primary_keys(default_conn) # Fetch the applied filter. filter_applied = trans_obj.is_filter_applied() @@ -303,6 +322,8 @@ def start_query_tool(trans_id): # Use pickle.loads function to get the command object session_obj = grid_data[str(trans_id)] trans_obj = pickle.loads(session_obj['command_obj']) + # set fetched row count to 0 as we are executing query again. + trans_obj.update_fetched_row_cnt(0) can_edit = False can_filter = False @@ -429,65 +450,6 @@ def preferences(trans_id): return success_return() -@blueprint.route('/columns/', methods=["GET"]) -@login_required -def get_columns(trans_id): - """ - This method will returns list of columns of last async query. - - Args: - trans_id: unique transaction id - """ - columns = dict() - columns_info = None - primary_keys = None - rset = None - status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id) - if status and conn is not None and session_obj is not None: - - ver = conn.manager.version - # Get the template path for the column - template_path = 'column/sql/#{0}#'.format(ver) - command_obj = pickle.loads(session_obj['command_obj']) - if hasattr(command_obj, 'obj_id'): - SQL = render_template("/".join([template_path, - 'nodes.sql']), - tid=command_obj.obj_id) - # rows with attribute not_null - status, rset = conn.execute_2darray(SQL) - if not status: - return internal_server_error(errormsg=rset) - - # Check PK column info is available or not - if 'primary_keys' in session_obj: - primary_keys = session_obj['primary_keys'] - - # Fetch column information - columns_info = conn.get_column_info() - if columns_info is not None: - for key, col in enumerate(columns_info): - col_type = dict() - col_type['type_code'] = col['type_code'] - col_type['type_name'] = None - if rset: - col_type['not_null'] = col['not_null'] = \ - rset['rows'][key]['not_null'] - - col_type['has_default_val'] = col['has_default_val'] = \ - rset['rows'][key]['has_default_val'] - - columns[col['name']] = col_type - - # As we changed the transaction object we need to - # restore it and update the session variable. - session_obj['columns_info'] = columns - update_session_grid_transaction(trans_id, session_obj) - - return make_json_response(data={'status': True, - 'columns': columns_info, - 'primary_keys': primary_keys}) - - @blueprint.route('/poll/', methods=["GET"]) @login_required def poll(trans_id): @@ -499,12 +461,21 @@ def poll(trans_id): """ result = None rows_affected = 0 + rows_fetched_from = 0 + rows_fetched_to = 0 + has_more_rows = False additional_result = [] + columns = dict() + columns_info = None + primary_keys = None + types = {} + client_primary_key = None + rset = None # Check the transaction and connection status status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id) if status and conn is not None and session_obj is not None: - status, result = conn.poll(formatted_exception_msg=True) + status, result = conn.poll(formatted_exception_msg=True, no_result=True) if not status: return internal_server_error(result) elif status == ASYNC_OK: @@ -519,6 +490,80 @@ def poll(trans_id): if (trans_status == TX_STATUS_INERROR and trans_obj.auto_rollback): conn.execute_void("ROLLBACK;") + + st, result = conn.async_fetchmany_2darray(ON_DEMAND_RECORD_COUNT) + if st: + if 'primary_keys' in session_obj: + primary_keys = session_obj['primary_keys'] + + # Fetch column information + columns_info = conn.get_column_info() + client_primary_key = generate_unique_client_primary_key( + columns_info + ) + session_obj['client_primary_key'] = client_primary_key + + if columns_info is not None: + + command_obj = pickle.loads(session_obj['command_obj']) + if hasattr(command_obj, 'obj_id'): + # Get the template path for the column + template_path = 'column/sql/#{0}#'.format( + conn.manager.version + ) + + SQL = render_template("/".join([template_path, + 'nodes.sql']), + tid=command_obj.obj_id) + # rows with attribute not_null + colst, rset = conn.execute_2darray(SQL) + if not colst: + return internal_server_error(errormsg=rset) + + for key, col in enumerate(columns_info): + col_type = dict() + col_type['type_code'] = col['type_code'] + col_type['type_name'] = None + columns[col['name']] = col_type + + if rset: + col_type['not_null'] = col['not_null'] = \ + rset['rows'][key]['not_null'] + + col_type['has_default_val'] = \ + col['has_default_val'] = \ + rset['rows'][key]['has_default_val'] + + if columns: + st, types = fetch_pg_types(columns, trans_obj) + + if not st: + return internal_server_error(types) + + for col_info in columns.values(): + for col_type in types: + if col_type['oid'] == col_info['type_code']: + col_info['type_name'] = col_type['typname'] + + session_obj['columns_info'] = columns + # status of async_fetchmany_2darray is True and result is none + # means nothing to fetch + if result and rows_affected > -1: + res_len = len(result) + if res_len == ON_DEMAND_RECORD_COUNT: + has_more_rows = True + + if res_len > 0: + rows_fetched_from = trans_obj.get_fetched_row_cnt() + trans_obj.update_fetched_row_cnt(rows_fetched_from + res_len) + rows_fetched_from += 1 + rows_fetched_to = trans_obj.get_fetched_row_cnt() + session_obj['command_obj'] = pickle.dumps(trans_obj, -1) + + # As we changed the transaction object we need to + # restore it and update the session variable. + update_session_grid_transaction(trans_id, session_obj) + elif status == ASYNC_EXECUTION_ABORTED: status = 'Cancel' else: @@ -559,51 +604,123 @@ def poll(trans_id): data={ 'status': status, 'result': result, 'rows_affected': rows_affected, - 'additional_messages': additional_messages + 'rows_fetched_from': rows_fetched_from, + 'rows_fetched_to': rows_fetched_to, + 'additional_messages': additional_messages, + 'has_more_rows': has_more_rows, + 'colinfo': columns_info, + 'primary_keys': primary_keys, + 'types': types, + 'client_primary_key': client_primary_key } ) -@blueprint.route('/fetch/types/', methods=["GET"]) +@blueprint.route('/fetch/', methods=["GET"]) +@blueprint.route('/fetch//', methods=["GET"]) @login_required -def fetch_pg_types(trans_id): +def fetch(trans_id, fetch_all=None): + result = None + has_more_rows = False + rows_fetched_from = 0 + rows_fetched_to = 0 + fetch_row_cnt = -1 if fetch_all == 1 else ON_DEMAND_RECORD_COUNT + + # Check the transaction and connection status + status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id) + if status and conn is not None and session_obj is not None: + status, result = conn.async_fetchmany_2darray(fetch_row_cnt) + if not status: + status = 'Error' + else: + status = 'Success' + res_len = len(result) + if fetch_row_cnt != -1 and res_len == ON_DEMAND_RECORD_COUNT: + has_more_rows = True + + if res_len: + rows_fetched_from = trans_obj.get_fetched_row_cnt() + trans_obj.update_fetched_row_cnt(rows_fetched_from + res_len) + rows_fetched_from += 1 + rows_fetched_to = trans_obj.get_fetched_row_cnt() + session_obj['command_obj'] = pickle.dumps(trans_obj, -1) + update_session_grid_transaction(trans_id, session_obj) + else: + status = 'NotConnected' + result = error_msg + + return make_json_response( + data={ + 'status': status, 'result': result, + 'has_more_rows': has_more_rows, + 'rows_fetched_from': rows_fetched_from, + 'rows_fetched_to': rows_fetched_to + } + ) + + +def fetch_pg_types(columns_info, trans_obj): """ This method is used to fetch the pg types, which is required to map the data type comes as a result of the query. Args: - trans_id: unique transaction id + columns_info: """ - # Check the transaction and connection status - status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id) - if status and conn is not None \ - and trans_obj is not None and session_obj is not None: - res = {} - if 'columns_info' in session_obj \ - and session_obj['columns_info'] is not None: + # get the default connection as current connection attached to trans id + # holds the cursor which has query result so we cannot use that connection + # to execute another query otherwise we'll lose query result. + + manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(trans_obj.sid) + default_conn = manager.connection(did=trans_obj.did) + + # Connect to the Server if not connected. + res = [] + if not default_conn.connected(): + status, msg = default_conn.connect() + if not status: + return status, msg - oids = [session_obj['columns_info'][col]['type_code'] for col in session_obj['columns_info']] + oids = [columns_info[col]['type_code'] for col in columns_info] - if oids: - status, res = conn.execute_dict( - u"""SELECT oid, format_type(oid,null) as typname FROM pg_type WHERE oid IN %s ORDER BY oid; + if oids: + status, res = default_conn.execute_dict( + u"""SELECT oid, format_type(oid,null) as typname FROM pg_type WHERE oid IN %s ORDER BY oid; """, [tuple(oids)]) - if status: - # iterate through pg_types and update the type name in session object - for record in res['rows']: - for col in session_obj['columns_info']: - type_obj = session_obj['columns_info'][col] - if type_obj['type_code'] == record['oid']: - type_obj['type_name'] = record['typname'] + if not status: + return False, res - update_session_grid_transaction(trans_id, session_obj) + return status, res['rows'] else: - status = False - res = error_msg - - return make_json_response(data={'status': status, 'result': res}) + return True, [] + + +def generate_unique_client_primary_key(columns_info): + temp_key = '__temp_PK' + if not columns_info: + return temp_key + + initial_temp_key_len = len(temp_key) + duplicate = False + suffix = 1 + while 1: + for col in columns_info: + if col['name'] == temp_key: + duplicate = True + break + if duplicate: + if initial_temp_key_len == len(temp_key): + temp_key += str(suffix) + suffix += 1 + else: + temp_key = temp_key[:-1] + str(suffix) + suffix += 1 + duplicate = False + else: + break + return temp_key @blueprint.route('/save/', methods=["PUT", "POST"]) @@ -615,7 +732,6 @@ def save(trans_id): Args: trans_id: unique transaction id """ - if request.data: changed_data = json.loads(request.data, encoding='utf-8') else: @@ -625,7 +741,6 @@ def save(trans_id): status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id) if status and conn is not None \ and trans_obj is not None and session_obj is not None: - setattr(trans_obj, 'columns_info', session_obj['columns_info']) # If there is no primary key found then return from the function. if len(session_obj['primary_keys']) <= 0 or len(changed_data) <= 0: @@ -636,7 +751,22 @@ def save(trans_id): } ) - status, res, query_res, _rowid = trans_obj.save(changed_data) + manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(trans_obj.sid) + default_conn = manager.connection(did=trans_obj.did) + + # Connect to the Server if not connected. + if not default_conn.connected(): + status, msg = default_conn.connect() + if not status: + return make_json_response( + data={'status': status, 'result': u"{}".format(msg)} + ) + + status, res, query_res, _rowid = trans_obj.save( + changed_data, + session_obj['columns_info'], + session_obj['client_primary_key'], + default_conn) else: status = False res = error_msg diff --git a/web/pgadmin/tools/sqleditor/command.py b/web/pgadmin/tools/sqleditor/command.py index b7d8a78..51b1f56 100644 --- a/web/pgadmin/tools/sqleditor/command.py +++ b/web/pgadmin/tools/sqleditor/command.py @@ -258,7 +258,21 @@ class SQLFilter(object): return status, result -class GridCommand(BaseCommand, SQLFilter): +class FetchedRowTracker(object): + """ + Keeps track of fetched row count. + """ + def __init__(self, **kwargs): + self.fetched_rows = 0 + + def get_fetched_row_cnt(self): + return self.fetched_rows + + def update_fetched_row_cnt(self, rows_cnt): + self.fetched_rows = rows_cnt + + +class GridCommand(BaseCommand, SQLFilter, FetchedRowTracker): """ class GridCommand(object) @@ -290,6 +304,7 @@ class GridCommand(BaseCommand, SQLFilter): """ BaseCommand.__init__(self, **kwargs) SQLFilter.__init__(self, **kwargs) + FetchedRowTracker.__init__(self, **kwargs) # Save the connection id, command type self.conn_id = kwargs['conn_id'] if 'conn_id' in kwargs else None @@ -299,10 +314,10 @@ class GridCommand(BaseCommand, SQLFilter): if self.cmd_type == VIEW_FIRST_100_ROWS or self.cmd_type == VIEW_LAST_100_ROWS: self.limit = 100 - def get_primary_keys(self): + def get_primary_keys(self, *args, **kwargs): return None, None - def save(self, changed_data): + def save(self, changed_data, default_conn=None): return forbidden(errmsg=gettext("Data cannot be saved for the current object.")) def get_limit(self): @@ -340,14 +355,14 @@ class TableCommand(GridCommand): # call base class init to fetch the table name super(TableCommand, self).__init__(**kwargs) - def get_sql(self): + def get_sql(self, default_conn=None): """ This method is used to create a proper SQL query to fetch the data for the specified table """ # Fetch the primary keys for the table - pk_names, primary_keys = self.get_primary_keys() + pk_names, primary_keys = self.get_primary_keys(default_conn) sql_filter = self.get_filter() @@ -362,13 +377,16 @@ class TableCommand(GridCommand): return sql - def get_primary_keys(self): + def get_primary_keys(self, default_conn=None): """ This function is used to fetch the primary key columns. """ driver = get_driver(PG_DEFAULT_DRIVER) - manager = driver.connection_manager(self.sid) - conn = manager.connection(did=self.did, conn_id=self.conn_id) + if default_conn is None: + manager = driver.connection_manager(self.sid) + conn = manager.connection(did=self.did, conn_id=self.conn_id) + else: + conn = default_conn pk_names = '' primary_keys = OrderedDict() @@ -400,7 +418,11 @@ class TableCommand(GridCommand): def can_filter(self): return True - def save(self, changed_data): + def save(self, + changed_data, + columns_info, + client_primary_key='__temp_PK', + default_conn=None): """ This function is used to save the data into the database. Depending on condition it will either update or insert the @@ -408,10 +430,16 @@ class TableCommand(GridCommand): Args: changed_data: Contains data to be saved + columns_info: + default_conn: + client_primary_key: """ - - manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(self.sid) - conn = manager.connection(did=self.did, conn_id=self.conn_id) + driver = get_driver(PG_DEFAULT_DRIVER) + if default_conn is None: + manager = driver.connection_manager(self.sid) + conn = manager.connection(did=self.did, conn_id=self.conn_id) + else: + conn = default_conn status = False res = None @@ -421,14 +449,6 @@ class TableCommand(GridCommand): list_of_sql = [] _rowid = None - # Replace column positions with names - def set_column_names(data): - new_data = {} - for key in data: - new_data[changed_data['columns'][int(key)]['name']] = data[key] - - return new_data - if conn.connected(): # Start the transaction @@ -454,28 +474,26 @@ class TableCommand(GridCommand): column_type = {} pk_names, primary_keys = self.get_primary_keys() - for each_col in self.columns_info: + for each_col in columns_info: if ( - self.columns_info[each_col]['not_null'] and - not self.columns_info[each_col][ + columns_info[each_col]['not_null'] and + not columns_info[each_col][ 'has_default_val'] ): column_data[each_col] = None column_type[each_col] =\ - self.columns_info[each_col]['type_name'] + columns_info[each_col]['type_name'] else: column_type[each_col] = \ - self.columns_info[each_col]['type_name'] - + columns_info[each_col]['type_name'] for each_row in changed_data[of_type]: data = changed_data[of_type][each_row]['data'] # Remove our unique tracking key - data.pop('__temp_PK', None) + data.pop(client_primary_key, None) data.pop('is_row_copied', None) - data = set_column_names(data) - data_type = set_column_names(changed_data[of_type][each_row]['data_type']) - list_of_rowid.append(data.get('__temp_PK')) + data_type = changed_data[of_type][each_row]['data_type'] + list_of_rowid.append(data.get(client_primary_key)) # Update columns value and data type # with columns having not_null=False and has @@ -497,9 +515,9 @@ class TableCommand(GridCommand): # For updated rows elif of_type == 'updated': for each_row in changed_data[of_type]: - data = set_column_names(changed_data[of_type][each_row]['data']) - pk = set_column_names(changed_data[of_type][each_row]['primary_keys']) - data_type = set_column_names(changed_data[of_type][each_row]['data_type']) + data = changed_data[of_type][each_row]['data'] + pk = changed_data[of_type][each_row]['primary_keys'] + data_type = changed_data[of_type][each_row]['data_type'] sql = render_template("/".join([self.sql_path, 'update.sql']), data_to_be_saved=data, primary_keys=pk, @@ -519,18 +537,19 @@ class TableCommand(GridCommand): rows_to_delete.append(changed_data[of_type][each_row]) # Fetch the keys for SQL generation if is_first: - # We need to covert dict_keys to normal list in Python3 - # In Python2, it's already a list & We will also fetch column names using index - keys = [ - changed_data['columns'][int(k)]['name'] - for k in list(changed_data[of_type][each_row].keys()) - ] + # We need to covert dict_keys to normal list in + # Python3 + # In Python2, it's already a list & We will also + # fetch column names using index + keys = list(changed_data[of_type][each_row].keys()) + no_of_keys = len(keys) is_first = False # Map index with column name for each row for row in rows_to_delete: for k, v in row.items(): - # Set primary key with label & delete index based mapped key + # Set primary key with label & delete index based + # mapped key try: row[changed_data['columns'][int(k)]['name']] = v except ValueError: @@ -597,7 +616,7 @@ class ViewCommand(GridCommand): # call base class init to fetch the table name super(ViewCommand, self).__init__(**kwargs) - def get_sql(self): + def get_sql(self, default_conn=None): """ This method is used to create a proper SQL query to fetch the data for the specified view @@ -652,7 +671,7 @@ class ForeignTableCommand(GridCommand): # call base class init to fetch the table name super(ForeignTableCommand, self).__init__(**kwargs) - def get_sql(self): + def get_sql(self, default_conn=None): """ This method is used to create a proper SQL query to fetch the data for the specified foreign table @@ -697,7 +716,7 @@ class CatalogCommand(GridCommand): # call base class init to fetch the table name super(CatalogCommand, self).__init__(**kwargs) - def get_sql(self): + def get_sql(self, default_conn=None): """ This method is used to create a proper SQL query to fetch the data for the specified catalog object @@ -722,7 +741,7 @@ class CatalogCommand(GridCommand): return True -class QueryToolCommand(BaseCommand): +class QueryToolCommand(BaseCommand, FetchedRowTracker): """ class QueryToolCommand(BaseCommand) @@ -732,13 +751,15 @@ class QueryToolCommand(BaseCommand): def __init__(self, **kwargs): # call base class init to fetch the table name - super(QueryToolCommand, self).__init__(**kwargs) + + BaseCommand.__init__(self, **kwargs) + FetchedRowTracker.__init__(self, **kwargs) self.conn_id = None self.auto_rollback = False self.auto_commit = True - def get_sql(self): + def get_sql(self, default_conn=None): return None def can_edit(self): diff --git a/web/pgadmin/tools/sqleditor/static/css/sqleditor.css b/web/pgadmin/tools/sqleditor/static/css/sqleditor.css index 76654d3..c48bb2c 100644 --- a/web/pgadmin/tools/sqleditor/static/css/sqleditor.css +++ b/web/pgadmin/tools/sqleditor/static/css/sqleditor.css @@ -435,6 +435,10 @@ input.editor-checkbox:focus { background-color: #e8e8e8; } +.sr .sc:first-child span { + float: right; +} + #datagrid div.slick-header.ui-state-default { background: #ffffff; border-bottom: none; @@ -455,4 +459,4 @@ input.editor-checkbox:focus { .sr.ui-widget-content { border-top: 1px solid silver; -} \ No newline at end of file +} diff --git a/web/pgadmin/tools/sqleditor/templates/sqleditor/js/sqleditor.js b/web/pgadmin/tools/sqleditor/templates/sqleditor/js/sqleditor.js index 8d835e3..6707acd 100644 --- a/web/pgadmin/tools/sqleditor/templates/sqleditor/js/sqleditor.js +++ b/web/pgadmin/tools/sqleditor/templates/sqleditor/js/sqleditor.js @@ -23,7 +23,8 @@ define([ 'slickgrid/plugins/slick.cellselectionmodel', 'slickgrid/plugins/slick.cellcopymanager', 'slickgrid/plugins/slick.rowselectionmodel', - 'slickgrid/slick.grid' + 'slickgrid/slick.grid', + 'slickgrid/slick.dataview' ], function( gettext, $, _, S, alertify, pgAdmin, Backbone, Backgrid, CodeMirror, pgExplain, GridSelector, clipboard, copyData, setStagedRows @@ -38,28 +39,6 @@ define([ pgBrowser = pgAdmin.Browser, Slick = window.Slick; - /* Reference link - * http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript - * Modified as per requirement. - */ - function epicRandomString(b) { - var s = []; - var hexDigits = "0123456789abcdef"; - for (var i = 0; i < 36; i++) { - s[i] = hexDigits.substr( - Math.floor(Math.random() * 0x10), 1 - ); - } - // bits 12-15 of the time_hi_and_version field to 0010 - s[14] = "4"; - // bits 6-7 of the clock_seq_hi_and_reserved to 01 - s[19] = hexDigits.substr((s[19] & 0x3) | 0x8, 1); - s[8] = s[13] = s[18] = s[23] = "-"; - - var uuid = s.join(""); - return uuid.replace(/-/g, '').substr(0, b); - }; - // Define key codes for shortcut keys var F5_KEY = 116, F7_KEY = 118, @@ -513,7 +492,7 @@ define([ - staged_rows: This will hold all the data which user copies/pastes/deletes in grid - deleted: - This will hold all the data which user delets in grid + This will hold all the data which user deletes in grid Events handling: ---------------- @@ -529,34 +508,10 @@ define([ - We are using this event for Copy operation on grid */ - // Get the item column value using a custom 'fieldIdx' column param - get_item_column_value: function (item, column) { - if (column.pos !== undefined) { - return item[column.pos]; - } else { - return null; - } - }, - // This function is responsible to create and render the SlickGrid. - render_grid: function(collection, columns, is_editable) { + render_grid: function(collection, columns, is_editable, client_primary_key) { var self = this; - // returns primary keys - self.handler.get_row_primary_key = function() { - var self = this, - tmp_keys = []; - _.each(self.primary_keys, function(p, idx) { - // For each columns search primary key position - _.each(self.columns, function(c) { - if(c.name == idx) { - tmp_keys.push(c.pos); - } - }); - }); - return tmp_keys; - }; - // This will work as data store and holds all the // inserted/updated/deleted data from grid self.handler.data_store = { @@ -571,8 +526,9 @@ define([ // To store primary keys before they gets changed self.handler.primary_keys_data = {}; - // Add getItemMetadata into handler for later use - self.handler.data_view = collection; + self.client_primary_key = client_primary_key; + + self.client_primary_key_counter = 0; // Remove any existing grid first if (self.handler.slickgrid) { @@ -622,7 +578,7 @@ define([ }); var gridSelector = new GridSelector(); - grid_columns = gridSelector.getColumnDefinitionsWithCheckboxes(grid_columns); + grid_columns = self.grid_columns = gridSelector.getColumnDefinitionsWithCheckboxes(grid_columns); var grid_options = { editable: true, @@ -630,8 +586,7 @@ define([ enableCellNavigation: true, enableColumnReorder: false, asyncEditorLoading: false, - autoEdit: false, - dataItemColumnValueExtractor: this.get_item_column_value + autoEdit: false }; var $data_grid = self.$el.find('#datagrid'); @@ -639,17 +594,16 @@ define([ var grid_height = $($('#editor-panel').find('.wcFrame')[1]).height() - 35; $data_grid.height(grid_height); - // Add our own custom primary key to keep track of changes - _.each(collection, function(row){ - row['__temp_PK'] = epicRandomString(15); - }); + var dataView = self.dataView = new Slick.Data.DataView(), + grid = self.grid = new Slick.Grid($data_grid, dataView, grid_columns, grid_options); + // Add-on function which allow us to identify the faulty row after insert/update // and apply css accordingly - collection.getItemMetadata = function(i) { - var res = {}, - cssClass = '', - data_store = self.handler.data_store; + + dataView.getItemMetadata = function(i) { + var res = {}, cssClass = '', + data_store = self.handler.data_store; if (_.has(self.handler, 'data_store')) { if (i in data_store.added_index && @@ -672,9 +626,8 @@ define([ cssClass += ' disabled_row'; } return {'cssClasses': cssClass}; - } + }; - var grid = new Slick.Grid($data_grid, collection, grid_columns, grid_options); grid.registerPlugin( new Slick.AutoTooltips({ enableForHeaderCells: false }) ); grid.setSelectionModel(new Slick.RowSelectionModel({selectActiveRow: false})); grid.registerPlugin(gridSelector); @@ -685,7 +638,8 @@ define([ columns: columns, grid: grid, selection: grid.getSelectionModel(), - editor: self + editor: self, + client_primary_key: self.client_primary_key }; self.handler.slickgrid = grid; @@ -696,6 +650,41 @@ define([ setStagedRows.bind(editor_data)); } + gridSelector.onBeforeGridSelectAll.subscribe(function(e, args) { + if (self.handler.has_more_rows) { + // this will prevent selection un-till we load all data + e.stopImmediatePropagation(); + self.fetch_next_all(function() { + // since we've stopped event propagation we need to + // trigger onGridSelectAll manually with new event data. + gridSelector.onGridSelectAll.notify(args, new Slick.EventData()); + }); + } + }); + + gridSelector.onBeforeGridColumnSelectAll.subscribe(function(e, args) { + if (self.handler.has_more_rows) { + // this will prevent selection un-till we load all data + e.stopImmediatePropagation(); + self.fetch_next_all(function() { + // since we've stopped event propagation we need to + // trigger onGridColumnSelectAll manually with new event data. + gridSelector.onGridColumnSelectAll.notify(args, new Slick.EventData()); + }); + } + }); + + // listen for row count change. + dataView.onRowCountChanged.subscribe(function (e, args) { + grid.updateRowCount(); + grid.render(); + }); + + // listen for rows change. + dataView.onRowsChanged.subscribe(function (e, args) { + grid.invalidateRows(args.rows); + grid.render(); + }); // Listener function which will be called before user updates existing cell // This will be used to collect primary key for that row @@ -709,8 +698,8 @@ define([ return false; } - if(self.handler.can_edit && before_data && '__temp_PK' in before_data) { - var _pk = before_data.__temp_PK, + if(self.handler.can_edit && before_data && self.client_primary_key in before_data) { + var _pk = before_data[self.client_primary_key], _keys = self.handler.primary_keys, current_pk = {}, each_pk_key = {}; @@ -722,8 +711,7 @@ define([ // Fetch primary keys for the row before they gets modified var _columns = self.handler.columns; _.each(_keys, function(value, key) { - pos = _.where(_columns, {name: key})[0]['pos'] - current_pk[pos] = before_data[pos]; + current_pk[key] = before_data[key]; }); // Place it in main variable for later use self.handler.primary_keys_data[_pk] = current_pk @@ -755,7 +743,7 @@ define([ // Fetch current row data from grid column_values = grid.getDataItem(row, cell) // Get the value from cell - value = column_values[column_info.pos] || ''; + value = column_values[column_info.field] || ''; // Copy this value to Clipboard if(value) clipboard.copyTextToClipboard(value); @@ -765,13 +753,12 @@ define([ }); - // Listener function which will be called when user updates existing rows grid.onCellChange.subscribe(function (e, args) { // self.handler.data_store.updated will holds all the updated data - var changed_column = args.grid.getColumns()[args.cell].pos, // Current field pos + var changed_column = args.grid.getColumns()[args.cell].field, // Current field pos updated_data = args.item[changed_column], // New value for current field - _pk = args.item.__temp_PK || null, // Unique key to identify row + _pk = args.item[self.client_primary_key] || null, // Unique key to identify row column_data = {}, _type; @@ -781,11 +768,16 @@ define([ // so that cell edit is enabled for that row. var grid = args.grid, row_data = grid.getDataItem(args.row), - p_keys_list = _.pick( - row_data, self.handler.get_row_primary_key() - ), - is_primary_key = Object.keys(p_keys_list).length ? - p_keys_list[0] : undefined; + is_primary_key = _.all( + _.values( + _.pick( + row_data, self.primary_keys + ) + ), + function(val) { + return val != undefined + } + ); // temp_new_rows is available only for view data. if (is_primary_key && self.handler.temp_new_rows) { @@ -805,7 +797,7 @@ define([ column_data); //Find type for current column self.handler.data_store.added[_pk]['err'] = false - self.handler.data_store.added[_pk]['data_type'][changed_column] = _.where(this.columns, {pos: changed_column})[0]['type']; + self.handler.data_store.added[_pk]['data_type'][changed_column] = _.where(this.columns, {name: changed_column})[0]['type']; // Check if it is updated data from existing rows? } else if(_pk in self.handler.data_store.updated) { _.extend( @@ -815,7 +807,7 @@ define([ self.handler.data_store.updated[_pk]['err'] = false //Find type for current column - self.handler.data_store.updated[_pk]['data_type'][changed_column] = _.where(this.columns, {pos: changed_column})[0]['type']; + self.handler.data_store.updated[_pk]['data_type'][changed_column] = _.where(this.columns, {name: changed_column})[0]['type']; } else { // First updated data for this primary key self.handler.data_store.updated[_pk] = { @@ -825,7 +817,7 @@ define([ self.handler.data_store.updated_index[args.row] = _pk; // Find & add column data type for current changed column var temp = {}; - temp[changed_column] = _.where(this.columns, {pos: changed_column})[0]['type']; + temp[changed_column] = _.where(this.columns, {name: changed_column})[0]['type']; self.handler.data_store.updated[_pk]['data_type'] = temp; } } @@ -844,11 +836,10 @@ define([ // Listener function which will be called when user adds new rows grid.onAddNewRow.subscribe(function (e, args) { // self.handler.data_store.added will holds all the newly added rows/data - var _key = epicRandomString(10), - column = args.column, - item = args.item, - data_length = this.grid.getDataLength(), - new_collection = args.grid.getData(); + var column = args.column, + item = args.item, data_length = this.grid.getDataLength(), + _key = (self.client_primary_key_counter++).toString(), + dataView = this.grid.getData(); // Add new row in list to keep track of it if (_.isUndefined(item[0])) { @@ -857,17 +848,16 @@ define([ // If copied item has already primary key, use it. if(item) { - item.__temp_PK = _key; + item[self.client_primary_key] = _key; } - new_collection.push(item); + dataView.addItem(item); self.handler.data_store.added[_key] = {'err': false, 'data': item}; self.handler.data_store.added_index[data_length] = _key; // Fetch data type & add it for the column var temp = {}; - temp[column.pos] = _.where(this.columns, {pos: column.pos})[0]['type']; + temp[column.name] = _.where(this.columns, {pos: column.pos})[0]['type']; self.handler.data_store.added[_key]['data_type'] = temp; - grid.invalidateRows([new_collection.length - 1]); grid.updateRowCount(); grid.render(); @@ -880,6 +870,16 @@ define([ $("#btn-save").prop('disabled', false); }.bind(editor_data)); + // Listen grid viewportChanged event to load next chunk of data. + grid.onViewportChanged.subscribe(function(e, args) { + var rendered_range = args.grid.getRenderedRange(), + data_len = args.grid.getDataLength(); + // start fetching next batch of records before reaching to bottom. + if (self.handler.has_more_rows && !self.handler.fetching_rows && rendered_range.bottom > data_len - 100) { + // fetch asynchronous + setTimeout(self.fetch_next.bind(self)); + } + }) // Resize SlickGrid when window resize $( window ).resize( function() { // Resize grid only when 'Data Output' panel is visible. @@ -902,6 +902,84 @@ define([ if(self.data_output_panel.isVisible()) self.grid_resize(grid); }); + + for (var i = 0; i < collection.length; i++) { + // Convert to dict from 2darray + var item = {}; + for (var j = 1; j < grid_columns.length; j++) { + item[grid_columns[j]['field']] = collection[i][grid_columns[j]['pos']] + } + + item[self.client_primary_key] = (self.client_primary_key_counter++).toString(); + collection[i] = item; + } + dataView.setItems(collection, self.client_primary_key); + }, + fetch_next_all(cb) { + this.fetch_next(true, cb); + }, + fetch_next: function(fetch_all, cb) { + var self = this; + + // This will prevent fetch operation if previous fetch operation is + // already in progress. + self.fetching_rows = true; + + $("#btn-flash").prop('disabled', true); + + if (fetch_all) { + self.handler.trigger( + 'pgadmin-sqleditor:loading-icon:show', + "{{ _('Fetching all records...') }}" + ); + } + + $.ajax({ + url: "{{ url_for('sqleditor.index') }}" + "fetch/" + self.transId + (fetch_all ? "/1": ""), + method: 'GET', + success: function(res) { + self.handler.has_more_rows = res.data.has_more_rows; + $("#btn-flash").prop('disabled', false); + self.handler.trigger('pgadmin-sqleditor:loading-icon:hide'); + self.update_grid_data(res.data.result); + if (typeof cb == "function") { + cb(); + } + }, + error: function(e) { + $("#btn-flash").prop('disabled', false); + self.handler.trigger('pgadmin-sqleditor:loading-icon:hide'); + self.handler.has_more_rows = false; + self.handler.fetching_rows = false; + if (typeof cb == "function") { + cb(); + } + if (e.readyState == 0) { + self.update_msg_history(false, + "{{ _('Not connected to the server or the connection to the server has been closed.') }}" + ); + return; + } + } + }); + }, + + update_grid_data: function(data) { + this.dataView.beginUpdate(); + + for (var i = 0; i < data.length; i++) { + // Convert 2darray to dict. + var item = {}; + for (var j = 1; j < this.grid_columns.length; j++) { + item[this.grid_columns[j]['field']] = data[i][this.grid_columns[j]['pos']] + } + + item[this.client_primary_key] = (this.client_primary_key_counter++).toString(); + this.dataView.addItem(item); + } + + this.dataView.endUpdate(); + this.handler.fetching_rows = false; }, /* This function is responsible to render output grid */ @@ -1576,6 +1654,8 @@ define([ self.explain_buffers = false; self.explain_timing = false; self.is_new_browser_tab = is_new_browser_tab; + self.has_more_rows = false; + self.fetching_rows = false; // We do not allow to call the start multiple times. if (self.gridView) @@ -1683,6 +1763,8 @@ define([ self.rows_to_disable = new Array(); // Temporarily hold new rows added self.temp_new_rows = new Array(); + self.has_more_rows = false; + self.fetching_rows = false; self.trigger( 'pgadmin-sqleditor:loading-icon:show', @@ -1761,45 +1843,14 @@ define([ }); }, - // This function makes the ajax call to fetch columns for last async query, - get_columns: function(poll_result) { - var self = this; - // Check the flag and decide if we need to fetch columns from server - // or use the columns data stored locally from previous call? - if (self.FETCH_COLUMNS_FROM_SERVER) { - $.ajax({ - url: "{{ url_for('sqleditor.index') }}" + "columns/" + self.transId, - method: 'GET', - success: function(res) { - poll_result.colinfo = res.data.columns; - poll_result.primary_keys = res.data.primary_keys; - self.call_render_after_poll(poll_result); - // Set a flag to get columns to false & set the value for future use - self.FETCH_COLUMNS_FROM_SERVER = false; - self.COLUMNS_DATA = res; - }, - error: function(e) { - var msg = e.responseText; - if (e.responseJSON != undefined && e.responseJSON.errormsg != undefined) - msg = e.responseJSON.errormsg; - alertify.error(msg, 5); - } - }); - } else { - // Use the previously saved columns data - poll_result.colinfo = self.COLUMNS_DATA.data.columns; - poll_result.primary_keys = self.COLUMNS_DATA.data.primary_keys; - self.call_render_after_poll(poll_result); - } - }, - // This is a wrapper to call _render function // We need this because we have separated columns route & result route // We need to combine both result here in wrapper before rendering grid call_render_after_poll: function(res) { var self = this; self.query_end_time = new Date(); - self.rows_affected = res.rows_affected; + self.rows_affected = res.rows_affected, + self.has_more_rows = res.has_more_rows; /* If no column information is available it means query runs successfully with no result to display. In this @@ -1848,7 +1899,8 @@ define([ 'pgadmin-sqleditor:loading-icon:message', gettext("Loading data from the database server and rendering...") ); - self.get_columns(res.data); + + self.call_render_after_poll(res.data); } else if (res.data.status === 'Busy') { // If status is Busy then poll the result by recursive call to the poll function @@ -1908,6 +1960,7 @@ define([ var self = this; self.colinfo = data.col_info; self.primary_keys = data.primary_keys; + self.client_primary_key = data.client_primary_key; self.cell_selected = false; self.selected_model = null; self.changedModels = []; @@ -1996,7 +2049,8 @@ define([ setTimeout( function() { self.gridView.render_grid( - explain_data_array, self.columns, self.can_edit + explain_data_array, self.columns, self.can_edit, + self.client_primary_key ); // Make sure - the 'Explain' panel is visible, before - we // start rendering the grid. @@ -2012,7 +2066,8 @@ define([ self.gridView.data_output_panel.focus(); setTimeout( function() { - self.gridView.render_grid(data.result, self.columns, self.can_edit); + self.gridView.render_grid(data.result, self.columns, + self.can_edit, self.client_primary_key); }, 10 ); } @@ -2020,134 +2075,114 @@ define([ // Hide the loading icon self.trigger('pgadmin-sqleditor:loading-icon:hide'); $("#btn-flash").prop('disabled', false); - }.bind(self), - function() { - this.trigger('pgadmin-sqleditor:loading-icon:hide'); - $("#btn-flash").prop('disabled', false); }.bind(self) ); }, // This function creates the columns as required by the backgrid - _fetch_column_metadata: function(data, cb, _fail) { + _fetch_column_metadata: function(data, cb) { var colinfo = data.colinfo, primary_keys = data.primary_keys, result = data.result, columns = [], self = this; + // Store pg_types in an array + var pg_types = new Array(); + _.each(data.types, function(r) { + pg_types[r.oid] = [r.typname]; + }); - self.trigger( - 'pgadmin-sqleditor:loading-icon:message', - gettext("Retrieving information about the columns returned...") - ); + // Create columns required by slick grid to render + _.each(colinfo, function(c) { + var is_primary_key = false; - // Make ajax call to fetch the pg types to map numeric data type - $.ajax({ - url: "{{ url_for('sqleditor.index') }}" + "fetch/types/" + self.transId, - method: 'GET', - success: function(res) { - if (res.data.status) { - // Store pg_types in an array - var pg_types = new Array(); - _.each(res.data.result.rows, function(r) { - pg_types[r.oid] = [r.typname]; - }); + // Check whether table have primary key + if (_.size(primary_keys) > 0) { + _.each(primary_keys, function (value, key) { + if (key === c.name) + is_primary_key = true; + }); + } - // Create columns required by backgrid to render - _.each(colinfo, function(c) { - var is_primary_key = false; + // To show column label and data type in multiline, + // The elements should be put inside the div. + // Create column label and type. + var col_type = column_label = ''; + var type = pg_types[c.type_code] ? + pg_types[c.type_code][0] : + // This is the case where user might + // have use casting so we will use type + // returned by cast function + pg_types[pg_types.length - 1][0] ? + pg_types[pg_types.length - 1][0] : 'unknown'; + + if (!is_primary_key) + col_type += ' ' + type; + else + col_type += ' [PK] ' + type; - // Check whether table have primary key - if (_.size(primary_keys) > 0) { - _.each(primary_keys, function (value, key) { - if (key === c.name) - is_primary_key = true; - }); - } + if (c.precision && c.precision >= 0 && c.precision != 65535) { + col_type += ' (' + c.precision; + col_type += c.scale && c.scale != 65535 ? + ',' + c.scale + ')': + ')'; + } - // To show column label and data type in multiline, - // The elements should be put inside the div. - // Create column label and type. - var col_type = column_label = ''; - var type = pg_types[c.type_code] ? - pg_types[c.type_code][0] : - // This is the case where user might - // have use casting so we will use type - // returned by cast function - pg_types[pg_types.length - 1][0] ? - pg_types[pg_types.length - 1][0] : 'unknown'; - - if (!is_primary_key) - col_type += ' ' + type; - else - col_type += ' [PK] ' + type; + // Identify cell type of column. + switch(type) { + case "json": + case "json[]": + case "jsonb": + case "jsonb[]": + col_cell = 'Json'; + break; + case "smallint": + case "integer": + case "bigint": + case "decimal": + case "numeric": + case "real": + case "double precision": + col_cell = 'number'; + break; + case "boolean": + col_cell = 'boolean'; + break; + case "character": + case "character[]": + case "character varying": + case "character varying[]": + if (c.internal_size && c.internal_size >= 0 && c.internal_size != 65535) { + // Update column type to display length on column header + col_type += ' (' + c.internal_size + ')'; + } + col_cell = 'string'; + break; + default: + col_cell = 'string'; + } - if (c.precision && c.precision >= 0 && c.precision != 65535) { - col_type += ' (' + c.precision; - col_type += c.scale && c.scale != 65535 ? - ',' + c.scale + ')': - ')'; - } + column_label = c.display_name + '
' + col_type; + + var col = { + 'name': c.name, + 'pos': c.pos, + 'label': column_label, + 'cell': col_cell, + 'can_edit': self.can_edit, + 'type': type, + 'not_null': c.not_null, + 'has_default_val': c.has_default_val + }; + columns.push(col); + }); + + self.columns = columns; + if (cb && typeof(cb) == 'function') { + cb(); + } - // Identify cell type of column. - switch(type) { - case "json": - case "json[]": - case "jsonb": - case "jsonb[]": - col_cell = 'Json'; - break; - case "smallint": - case "integer": - case "bigint": - case "decimal": - case "numeric": - case "real": - case "double precision": - col_cell = 'number'; - break; - case "boolean": - col_cell = 'boolean'; - break; - case "character": - case "character[]": - case "character varying": - case "character varying[]": - if (c.internal_size && c.internal_size >= 0 && c.internal_size != 65535) { - // Update column type to display length on column header - col_type += ' (' + c.internal_size + ')'; - } - col_cell = 'string'; - break; - default: - col_cell = 'string'; - } - column_label = c.display_name + '
' + col_type; - - var col = { - 'name': c.name, - 'pos': c.pos, - 'label': column_label, - 'cell': col_cell, - 'can_edit': self.can_edit, - 'type': type, - 'not_null': c.not_null, - 'has_default_val': c.has_default_val - }; - columns.push(col); - }); - } - else { - alertify.alert('Fetching Type Error', res.data.result); - } - self.columns = columns; - if (cb && typeof(cb) == 'function') { - cb(); - } - }, - fail: _fail - }); }, // This function is used to raise appropriate message. @@ -2158,22 +2193,21 @@ define([ self.gridView.messages_panel.focus(); - if (self.is_query_tool) { - if (clear_grid) { - // Delete grid - if (self.gridView.handler.slickgrid) { - self.gridView.handler.slickgrid.destroy(); + if (clear_grid) { + // Delete grid + if (self.gridView.handler.slickgrid) { + self.gridView.handler.slickgrid.destroy(); - } - // Misc cleaning - self.columns = undefined; - self.collection = undefined; - - $('.sql-editor-message').text(msg); - } else { - $('.sql-editor-message').append(msg); } + // Misc cleaning + self.columns = undefined; + self.collection = undefined; + + $('.sql-editor-message').text(msg); + } else { + $('.sql-editor-message').append(msg); } + // Scroll automatically when msgs appends to element setTimeout(function(){ $(".sql-editor-message").scrollTop($(".sql-editor-message")[0].scrollHeight);; @@ -2225,7 +2259,7 @@ define([ rows_to_delete: function(data) { var self = this, - tmp_keys = self.get_row_primary_key.call(self); + tmp_keys = self.primary_keys; // re-calculate rows with no primary keys self.temp_new_rows = []; @@ -2238,7 +2272,6 @@ define([ self.temp_new_rows.push(idx); } }); - data.getItemMetadata = self.data_view.getItemMetadata; self.rows_to_disable = _.clone(self.temp_new_rows); }, @@ -2249,69 +2282,73 @@ define([ is_added = _.size(self.data_store.added), is_updated = _.size(self.data_store.updated); - // Remove newly added rows from staged rows as we don't want to send them on server - if(is_added) { - _.each(self.data_store.added, function(val, key) { - if(key in self.data_store.staged_rows) { - // Remove the row from data store so that we do not send it on server - deleted_keys.push(key); - delete self.data_store.staged_rows[key]; - delete self.data_store.added[key] - } - }); + // Remove newly added rows from staged rows as we don't want to send them on server + if(is_added) { + _.each(self.data_store.added, function(val, key) { + if(key in self.data_store.staged_rows) { + // Remove the row from data store so that we do not send it on server + deleted_keys.push(key); + delete self.data_store.staged_rows[key]; + delete self.data_store.added[key]; + delete self.data_store.added_index[key]; } - - // If only newly rows to delete and no data is there to send on server - // then just re-render the grid - if(_.size(self.data_store.staged_rows) == 0) { - var grid = self.slickgrid, data = grid.getData(), idx = 0; - if(deleted_keys.length){ - // Remove new rows from grid data using deleted keys - data = _.reject(data, function(d){ - return (d && _.indexOf(deleted_keys, d.__temp_PK) > -1) - }); - } - self.rows_to_delete.apply(self, [data]); - grid.resetActiveCell(); - grid.setData(data, true); - grid.setSelectedRows([]); - grid.invalidate(); - // Nothing to copy or delete here - $("#btn-delete-row").prop('disabled', true); - $("#btn-copy-row").prop('disabled', true); - if(_.size(self.data_store.added) || is_updated) { - // Do not disable save button if there are - // any other changes present in grid data - $("#btn-save").prop('disabled', false); - } else { - $("#btn-save").prop('disabled', true); - } - alertify.success(gettext("Row(s) deleted")); + }); + } + // If only newly rows to delete and no data is there to send on server + // then just re-render the grid + if(_.size(self.data_store.staged_rows) == 0) { + var grid = self.slickgrid, + dataView = grid.getData(), + data = dataView.getItems(), + idx = 0; + + grid.resetActiveCell(); + + dataView.beginUpdate(); + for (var i = 0; i < deleted_keys.length; i++) { + dataView.deleteItem(deleted_keys[i]); + } + dataView.endUpdate(); + self.rows_to_delete.apply(self, [dataView.getItems()]); + grid.resetActiveCell(); + grid.setSelectedRows([]); + grid.invalidate(); + + // Nothing to copy or delete here + $("#btn-delete-row").prop('disabled', true); + $("#btn-copy-row").prop('disabled', true); + if(_.size(self.data_store.added) || is_updated) { + // Do not disable save button if there are + // any other changes present in grid data + $("#btn-save").prop('disabled', false); } else { - // There are other data to needs to be updated on server - if(is_updated) { - alertify.alert(gettext("Operation failed"), + $("#btn-save").prop('disabled', true); + } + alertify.success(gettext("Row(s) deleted")); + } else { + // There are other data to needs to be updated on server + if(is_updated) { + alertify.alert(gettext("Operation failed"), gettext("There are unsaved changes in grid, Please save them first to avoid inconsistency in data") ); - return; - } - alertify.confirm(gettext("Delete Row(s)"), + return; + } + alertify.confirm(gettext("Delete Row(s)"), gettext("Are you sure you wish to delete selected row(s)?"), - function() { - $("#btn-delete-row").prop('disabled', true); - $("#btn-copy-row").prop('disabled', true); - // Change the state - self.data_store.deleted = self.data_store.staged_rows; - self.data_store.staged_rows = {}; - // Save the changes on server - self._save(); - }, - function() { - // Do nothing as user canceled the operation. - } - ).set('labels', {ok:'Yes', cancel:'No'}); + function() { + $("#btn-delete-row").prop('disabled', true); + $("#btn-copy-row").prop('disabled', true); + // Change the state + self.data_store.deleted = self.data_store.staged_rows; + self.data_store.staged_rows = {}; + // Save the changes on server + self._save(); + }, + function() { + // Do nothing as user canceled the operation. } - + ).set('labels', {ok: gettext("Yes"), cancel:gettext("No")}); + } }, /* This function will fetch the list of changed models and make @@ -2373,7 +2410,9 @@ define([ data: JSON.stringify(req_data), success: function(res) { var grid = self.slickgrid, - data = grid.getData(); + dataView = grid.getData(), + data_length = dataView.getLength(), + data = []; if (res.data.status) { // Remove flag is_row_copied from copied rows _.each(data, function(row, idx) { @@ -2390,22 +2429,20 @@ define([ // Remove deleted rows from client as well if(is_deleted) { var rows = grid.getSelectedRows(); - /* In JavaScript sorting by default is lexical, - * To make sorting numerical we need to pass function - * After that we will Reverse the order of sorted array - * so that when we remove it does not affect array index - */ - if(data.length == rows.length) { + if(data_length == rows.length) { // This means all the rows are selected, clear all data data = []; + dataView.setItems(data, self.client_primary_key); } else { - rows = rows.sort(function(a,b){return a - b}).reverse(); - rows.forEach(function(idx) { - data.splice(idx, 1); - }); + dataView.beginUpdate(); + for (var i = 0; i < rows.length; i++) { + item = grid.getDataItem(rows[i]); + data.push(item); + dataView.deleteItem(item[self.client_primary_key]); + } + dataView.endUpdate(); } self.rows_to_delete.apply(self, [data]); - grid.setData(data, true); grid.setSelectedRows([]); } @@ -2418,6 +2455,7 @@ define([ self.rows_to_disable = _.clone(self.temp_new_rows); } + grid.setSelectedRows([]); // Reset data store self.data_store = { 'added': {}, @@ -2438,7 +2476,7 @@ define([ $('.sql-editor-message').text(res.data.result); var err_msg = S(gettext("%s.")).sprintf(res.data.result).value(); alertify.notify(err_msg, 'error', 20); - + grid.setSelectedRows([]); // To highlight the row at fault if(_.has(res.data, '_rowid') && (!_.isUndefined(res.data._rowid)|| !_.isNull(res.data._rowid))) { @@ -2491,14 +2529,21 @@ define([ // Find index of row at fault from grid data _find_rowindex: function(rowid) { - var self = this; - var grid = self.slickgrid, - data = grid.getData(), _rowid, count = 0, _idx = -1; + var self = this, + grid = self.slickgrid, + dataView = grid.getData(), + data = dataView.getItems(), + _rowid, + count = 0, + _idx = -1; + // If _rowid is object then it's update/delete operation if(_.isObject(rowid)) { _rowid = rowid; - } else if (_.isString(rowid)) { // Insert opration - _rowid = { '__temp_PK': rowid }; + } else if (_.isString(rowid)) { // Insert operation + var rowid = {}; + rowid[self.client_primary_key]= rowid; + _rowid = rowid; } else { // Something is wrong with unique id return _idx; @@ -2720,11 +2765,6 @@ define([ // This function will set the required flag for polling response data _init_polling_flags: function() { var self = this; - // Set a flag to get columns - self.FETCH_COLUMNS_FROM_SERVER = true; - // We will set columns data in this variable for future use once we fetch it - // from server - self.COLUMNS_DATA = {}; // To get a timeout for polling fallback timer in seconds in // regards to elapsed time @@ -2820,7 +2860,7 @@ define([ return; // Add column position and it's value to data - data[column_info.field] = _values[column_info.pos] || ''; + data[column_info.field] = _values[column_info.field] || ''; self.trigger( 'pgadmin-sqleditor:loading-icon:show', @@ -2890,7 +2930,7 @@ define([ return; // Add column position and it's value to data - data[column_info.field] = _values[column_info.pos] || ''; + data[column_info.field] = _values[column_info.field] || ''; self.trigger( 'pgadmin-sqleditor:loading-icon:show', @@ -3053,16 +3093,18 @@ define([ _paste_row: function() { var self = this, col_info = {}, grid = self.slickgrid, - data = grid.getData(), - count = Object.keys(data).length-1; - - var rows = grid.getSelectedRows().sort( + dataView = grid.getData(), + data = dataView.getItems(), + count = dataView.getLength(), + rows = grid.getSelectedRows().sort( function (a, b) { return a - b; } ), - rows = rows.length == 0 ? self.last_copied_rows : rows, copied_rows = rows.map(function (rowIndex) { return data[rowIndex]; }); + + rows = rows.length == 0 ? self.last_copied_rows : rows + self.last_copied_rows = rows; // If there are rows to paste? @@ -3182,6 +3224,9 @@ define([ sql = '', history_msg = ''; + self.has_more_rows = false; + self.fetching_rows = false; + /* If code is selected in the code mirror then execute * the selected part else execute the complete code. */ diff --git a/web/pgadmin/utils/driver/abstract.py b/web/pgadmin/utils/driver/abstract.py index 9b2363c..7db3e37 100644 --- a/web/pgadmin/utils/driver/abstract.py +++ b/web/pgadmin/utils/driver/abstract.py @@ -101,6 +101,12 @@ class BaseConnection(object): - Implement this method to execute the given query and returns the result as an array of dict (column name -> value) format. + * def async_fetchmany_2darray(records=-1, formatted_exception_msg=False): + - Implement this method to retrieve result of asynchronous connection and + polling with no_result flag set to True. + This returns the result as a 2 dimensional array. + If records is -1 then fetchmany will behave as fetchall. + * connected() - Implement this method to get the status of the connection. It should return True for connected, otherwise False @@ -133,7 +139,7 @@ class BaseConnection(object): - Implement this method to wait for asynchronous connection with timeout. This must be a non blocking call. - * poll(formatted_exception_msg) + * poll(formatted_exception_msg, no_result) - Implement this method to poll the data of query running on asynchronous connection. @@ -180,6 +186,10 @@ class BaseConnection(object): pass @abstractmethod + def async_fetchmany_2darray(self, records=-1, formatted_exception_msg=False): + pass + + @abstractmethod def connected(self): pass @@ -208,7 +218,7 @@ class BaseConnection(object): pass @abstractmethod - def poll(self, formatted_exception_msg=True): + def poll(self, formatted_exception_msg=True, no_result=False): pass @abstractmethod diff --git a/web/pgadmin/utils/driver/psycopg2/__init__.py b/web/pgadmin/utils/driver/psycopg2/__init__.py index 502cee4..9e4a565 100644 --- a/web/pgadmin/utils/driver/psycopg2/__init__.py +++ b/web/pgadmin/utils/driver/psycopg2/__init__.py @@ -1072,6 +1072,55 @@ Failed to execute query (execute_void) for the server #{server_id} - {conn_id} return True, {'columns': columns, 'rows': rows} + def async_fetchmany_2darray(self, records=2000, formatted_exception_msg=False): + """ + User should poll and check if status is ASYNC_OK before calling this + function + Args: + records: no of records to fetch. use -1 to fetchall. + formatted_exception_msg: + + Returns: + + """ + cur = self.__async_cursor + if not cur: + return False, gettext( + "Cursor could not be found for the async connection." + ) + + if self.conn.isexecuting(): + return False, gettext( + "Asynchronous query execution/operation underway." + ) + + if self.row_count > 0: + result = [] + # For DDL operation, we may not have result. + # + # Because - there is not direct way to differentiate DML and + # DDL operations, we need to rely on exception to figure + # that out at the moment. + try: + if records == -1: + res = cur.fetchall() + else: + res = cur.fetchmany(records) + for row in res: + new_row = [] + for col in self.column_info: + new_row.append(row[col['name']]) + result.append(new_row) + except psycopg2.ProgrammingError as e: + result = None + else: + # User performed operation which dose not produce record/s as + # result. + # for eg. DDL operations. + return True, None + + return True, result + def connected(self): if self.conn: if not self.conn.closed: @@ -1218,7 +1267,7 @@ Failed to reset the connection to the server due to following error: "poll() returned %s from _wait_timeout function" % state ) - def poll(self, formatted_exception_msg=False): + def poll(self, formatted_exception_msg=False, no_result=False): """ This function is a wrapper around connection's poll function. It internally uses the _wait_timeout method to poll the @@ -1228,6 +1277,7 @@ Failed to reset the connection to the server due to following error: Args: formatted_exception_msg: if True then function return the formatted exception message, otherwise error string. + no_result: If True then only poll status will be returned. """ cur = self.__async_cursor @@ -1283,23 +1333,23 @@ Failed to reset the connection to the server due to following error: pos += 1 self.row_count = cur.rowcount - - if cur.rowcount > 0: - result = [] - # For DDL operation, we may not have result. - # - # Because - there is not direct way to differentiate DML and - # DDL operations, we need to rely on exception to figure that - # out at the moment. - try: - for row in cur: - new_row = [] - for col in self.column_info: - new_row.append(row[col['name']]) - result.append(new_row) - - except psycopg2.ProgrammingError: - result = None + if not no_result: + if cur.rowcount > 0: + result = [] + # For DDL operation, we may not have result. + # + # Because - there is not direct way to differentiate DML and + # DDL operations, we need to rely on exception to figure + # that out at the moment. + try: + for row in cur: + new_row = [] + for col in self.column_info: + new_row.append(row[col['name']]) + result.append(new_row) + + except psycopg2.ProgrammingError: + result = None return status, result diff --git a/web/regression/feature_utils/pgadmin_page.py b/web/regression/feature_utils/pgadmin_page.py index 15b78d9..d690612 100644 --- a/web/regression/feature_utils/pgadmin_page.py +++ b/web/regression/feature_utils/pgadmin_page.py @@ -193,6 +193,20 @@ class PgadminPage: self._wait_for("spinner to disappear", spinner_has_disappeared) + def wait_for_query_tool_loading_indicator_to_disappear(self): + def spinner_has_disappeared(driver): + try: + driver.find_element_by_xpath( + "//*[@id='fetching_data' and @class='hide']" + ) + return False + except NoSuchElementException: + # wait for loading indicator disappear animation to complete. + time.sleep(0.5) + return True + + self._wait_for("spinner to disappear", spinner_has_disappeared) + def wait_for_app(self): def page_shows_app(driver): if driver.title == self.app_config.APP_NAME: diff --git a/web/regression/javascript/selection/copy_data_spec.js b/web/regression/javascript/selection/copy_data_spec.js index affad66..9f60eb7 100644 --- a/web/regression/javascript/selection/copy_data_spec.js +++ b/web/regression/javascript/selection/copy_data_spec.js @@ -4,19 +4,23 @@ define( "slickgrid/slick.rowselectionmodel", "sources/selection/copy_data", "sources/selection/clipboard", - "sources/selection/range_selection_helper" + "sources/selection/range_selection_helper", + + "slickgrid/slick.dataview", ], function ($, SlickGrid, RowSelectionModel, copyData, clipboard, RangeSelectionHelper) { describe('copyData', function () { var grid, sqlEditor; beforeEach(function () { - var data = [[1, "leopord", "12"], - [2, "lion", "13"], - [3, "puma", "9"]]; + var data = [{"id": 1, "brand":"leopord", "size":"12", "__temp_PK": '123'}, + {"id": 2, "brand":"lion", "size":"13", "__temp_PK": '456'}, + {"id": 3, "brand":"puma", "size":"9", "__temp_PK": '789'}], + dataView = new Slick.Data.DataView(); var columns = [{ name: "id", + field: "id", pos: 0, label: "id
numeric", cell: "number", @@ -24,6 +28,7 @@ define( type: "numeric" }, { name: "brand", + field: "brand", pos: 1, label: "flavor
character varying", cell: "string", @@ -31,33 +36,32 @@ define( type: "character varying" }, { name: "size", + field: "size", pos: 2, label: "size
numeric", cell: "number", can_edit: false, type: "numeric" } - ] - ; + ]; var gridContainer = $("
"); $("body").append(gridContainer); $("body").append(""); - grid = new Slick.Grid("#grid", data, columns, {}); + grid = new Slick.Grid("#grid", dataView, columns, {}); + dataView.setItems(data, "__temp_PK"); grid.setSelectionModel(new Slick.RowSelectionModel({selectActiveRow: false})); sqlEditor = {slickgrid: grid}; }); afterEach(function() { + grid.destroy(); $("body").remove('#grid'); $("body").remove('#btn-paste-row'); }); describe("when rows are selected", function () { beforeEach(function () { - grid.getSelectionModel().setSelectedRanges([ - RangeSelectionHelper.rangeForRow(grid, 0), - RangeSelectionHelper.rangeForRow(grid, 2)] - ); + grid.setSelectedRows([0,2]); }); it("copies them", function () { diff --git a/web/regression/javascript/selection/range_boundary_navigator_spec.js b/web/regression/javascript/selection/range_boundary_navigator_spec.js index 8376d0a..39ea4fb 100644 --- a/web/regression/javascript/selection/range_boundary_navigator_spec.js +++ b/web/regression/javascript/selection/range_boundary_navigator_spec.js @@ -123,27 +123,29 @@ define(['sources/selection/range_boundary_navigator'], function (rangeBoundaryNa describe("#rangesToCsv", function () { var data, columnDefinitions, ranges; beforeEach(function () { - data = [[1, "leopard", "12"], - [2, "lion", "13"], - [3, "cougar", "9"], - [4, "tiger", "10"]]; - columnDefinitions = [{name: 'id', pos: 0}, {name: 'animal', pos: 1}, {name: 'size', pos: 2}]; + data = [{"id":1, "animal":"leopard", "size":"12"}, + {"id":2, "animal":"lion", "size":"13"}, + {"id":3, "animal":"cougar", "size":"9"}, + {"id":4, "animal":"tiger", "size":"10"}]; + + columnDefinitions = [{name: 'id', field: 'id', pos: 0}, + {name: 'animal', field: 'animal', pos: 1}, + {name: 'size', field: 'size', pos: 2}]; ranges = [new Slick.Range(0, 0, 0, 2), new Slick.Range(3, 0, 3, 2)]; }); it("returns csv for the provided ranges", function () { var csvResult = rangeBoundaryNavigator.rangesToCsv(data, columnDefinitions, ranges); - expect(csvResult).toEqual("1,'leopard','12'\n4,'tiger','10'"); }); describe("when there is an extra column with checkboxes", function () { beforeEach(function () { - columnDefinitions = [{name: 'not-a-data-column'}, {name: 'id', pos: 0}, {name: 'animal', pos: 1}, { - name: 'size', - pos: 2 - }]; + columnDefinitions = [{name: 'not-a-data-column'}, + {name: 'id', field: 'id', pos: 0}, + {name: 'animal', field: 'animal', pos: 1}, + {name: 'size', field: 'size',pos: 2}]; ranges = [new Slick.Range(0, 0, 0, 3), new Slick.Range(3, 0, 3, 3)]; }); diff --git a/web/regression/javascript/selection/row_selector_spec.js b/web/regression/javascript/selection/row_selector_spec.js index 10697e6..41a1e2e 100644 --- a/web/regression/javascript/selection/row_selector_spec.js +++ b/web/regression/javascript/selection/row_selector_spec.js @@ -5,6 +5,8 @@ define( "sources/selection/row_selector", "slickgrid/slick.rowselectionmodel", "slickgrid", + + "slickgrid/slick.dataview" ], function ($, _, SlickGrid, RowSelector, RowSelectionModel, Slick) { describe("RowSelector", function () { @@ -25,13 +27,14 @@ define( }]; var rowSelector = new RowSelector(); + dataView = new Slick.Data.DataView(), data = []; for (var i = 0; i < 10; i++) { - data.push(['some-value-' + i, 'second value ' + i]); + data.push({'some-column-name':'some-value-' + i, 'second column':'second value ' + i}); } columnDefinitions = rowSelector.getColumnDefinitionsWithCheckboxes(columnDefinitions); - grid = new SlickGrid(container, data, columnDefinitions); - + grid = new SlickGrid(container, dataView, columnDefinitions); + dataView.setItems(data, "some-column-name"); rowSelectionModel = new RowSelectionModel(); grid.setSelectionModel(rowSelectionModel); grid.registerPlugin(rowSelector); diff --git a/web/regression/javascript/selection/set_staged_rows_spec.js b/web/regression/javascript/selection/set_staged_rows_spec.js index 11e293f..2b08b65 100644 --- a/web/regression/javascript/selection/set_staged_rows_spec.js +++ b/web/regression/javascript/selection/set_staged_rows_spec.js @@ -11,17 +11,20 @@ define([ "jquery", "underscore", "sources/selection/set_staged_rows", + "slickgrid/slick.dataview", ], function ($, _, SetStagedRows) { describe('when no full rows are selected', function () { var sqlEditorObj, deleteButton, copyButton; beforeEach(function () { - var gridSpy = jasmine.createSpyObj('gridSpy', ['getData', 'getCellNode']); - gridSpy.getData.and.returnValue([ - {0: 'one', 1: 'two', __temp_PK: '123'}, - {0: 'three', 1: 'four', __temp_PK: '456'}, - {0: 'five', 1: 'six', __temp_PK: '789'}, - {0: 'seven', 1: 'eight', __temp_PK: '432'} - ]); + var data = [{'a pk column': 'one', 'some column': 'two', '__temp_PK': '123'}, + {'a pk column': 'three', 'some column': 'four', '__temp_PK': '456'}, + {'a pk column': 'five', 'some column': 'six', '__temp_PK': '789'}, + {'a pk column': 'seven', 'some column': 'eight', '__temp_PK': '432'}], + dataView = new Slick.Data.DataView(), + gridSpy = jasmine.createSpyObj('gridSpy', ['getData', 'getCellNode']); + + dataView.setItems(data, "__temp_PK"); + gridSpy.getData.and.returnValue(dataView); deleteButton = $(''); copyButton = $(''); sqlEditorObj = { @@ -29,7 +32,7 @@ define([ editor: { handler: { data_store: { - staged_rows: {1: [1, 2]} + staged_rows: {'123': [1, 2]} } } } @@ -88,13 +91,16 @@ define([ describe('when getSelectedRows is present in the selection model', function () { var sqlEditorObj, gridSpy, deleteButton, copyButton; beforeEach(function () { + var data = [{'a pk column': 'one', 'some column': 'two', '__temp_PK': '123'}, + {'a pk column': 'three', 'some column': 'four', '__temp_PK': '456'}, + {'a pk column': 'five', 'some column': 'six', '__temp_PK': '789'}, + {'a pk column': 'seven', 'some column': 'eight', '__temp_PK': '432'}], + dataView = new Slick.Data.DataView(); + + dataView.setItems(data, '__temp_PK'); + gridSpy = jasmine.createSpyObj('gridSpy', ['getData', 'getCellNode']); - gridSpy.getData.and.returnValue([ - {0: 'one', 1: 'two', __temp_PK: '123'}, - {0: 'three', 1: 'four', __temp_PK: '456'}, - {0: 'five', 1: 'six', __temp_PK: '789'}, - {0: 'seven', 1: 'eight', __temp_PK: '432'} - ]); + gridSpy.getData.and.returnValue(dataView); var selectionSpy = jasmine.createSpyObj('selectionSpy', ['getSelectedRows', 'setSelectedRows']); selectionSpy.getSelectedRows.and.returnValue([1, 2]); @@ -117,13 +123,16 @@ define([ columns: [ { name: 'a pk column', + field: 'a pk column', pos: 0 }, { name: 'some column', + field: 'some column', pos: 1 } - ] + ], + client_primary_key: '__temp_PK' }; $('body').append(deleteButton); @@ -167,14 +176,14 @@ define([ describe('when table has primary keys', function () { beforeEach(function () { sqlEditorObj.keys = {'a pk column': 'varchar'}; - sqlEditorObj.editor.handler.data_store.staged_rows = {'456': {0: 'three'}}; + sqlEditorObj.editor.handler.data_store.staged_rows = {'456': {'a pk column': 'three'}}; }); describe('selected rows have primary key', function () { it('should set the staged rows correctly', function () { SetStagedRows.call(sqlEditorObj, {}, {}); expect(sqlEditorObj.editor.handler.data_store.staged_rows).toEqual( - {'456': {0: 'three'}, '789': {0: 'five'}}); + {'456': {'a pk column': 'three'}, '789': {'a pk column': 'five'}}); }); it('should not clear selected rows in Cell Selection Model', function () { @@ -186,12 +195,16 @@ define([ describe('selected rows missing primary key', function () { beforeEach(function () { - gridSpy.getData.and.returnValue([ - {0: 'one', 1: 'two', __temp_PK: '123'}, - {1: 'four', __temp_PK: '456'}, - {1: 'six', __temp_PK: '789'}, - {0: 'seven', 1: 'eight', __temp_PK: '432'} - ]); + + var data = [{'a pk column': 'one', 'some column': 'two', '__temp_PK': '123'}, + {'some column': 'four', '__temp_PK': '456'}, + {'some column': 'six', '__temp_PK': '789'}, + {'a pk column': 'seven', 'some column': 'eight', '__temp_PK': '432'}], + dataView = new Slick.Data.DataView(); + + dataView.setItems(data, '__temp_PK'); + + gridSpy.getData.and.returnValue(dataView); }); it('should clear the staged rows', function () { @@ -223,8 +236,8 @@ define([ it('should not clear the staged rows', function () { expect(sqlEditorObj.editor.handler.data_store.staged_rows).toEqual({ - '456': {0: 'three'}, - '789': {0: 'five'} + '456': {'a pk column': 'three'}, + '789': {'a pk column': 'five'} }); }); diff --git a/web/regression/javascript/test-main.js b/web/regression/javascript/test-main.js index 421bb17..20e68a2 100644 --- a/web/regression/javascript/test-main.js +++ b/web/regression/javascript/test-main.js @@ -36,6 +36,7 @@ require.config({ 'underscore.string': sourcesDir + 'vendor/underscore/underscore.string', 'slickgrid': sourcesDir + 'vendor/slickgrid/slick.core', 'slickgrid/slick.grid': sourcesDir + 'vendor/slickgrid/slick.grid', + 'slickgrid/slick.dataview': sourcesDir + 'vendor/slickgrid/slick.dataview', 'slickgrid/slick.rowselectionmodel': sourcesDir + 'vendor/slickgrid/plugins/slick.rowselectionmodel', 'translations': '/base/regression/javascript/fake_translations', 'sources': sourcesDir + 'js', @@ -58,6 +59,12 @@ require.config({ ], "exports": 'window.Slick.Grid' }, + "slickgrid/slick.dataview": { + "deps": [ + 'jquery', "jquery.ui", "jquery.event.drag", "slickgrid" + ], + "exports": 'window.Slick.Data.DataView' + }, "slickgrid/slick.rowselectionmodel": { "deps": [ "jquery"