diff --git a/docs/cyberark-pam-import.md b/docs/cyberark-pam-import.md index a95b16e00..86f34960a 100644 --- a/docs/cyberark-pam-import.md +++ b/docs/cyberark-pam-import.md @@ -62,8 +62,13 @@ Keeper Vault/ | PaloAltoNetworks, CiscoIOS, CiscoASA, JuniperJunos, F5BigIP, CheckPointGAIA | pamMachine | ssh | 22 | | CyberArk (internal) | pamMachine | ssh | 22 | | BusinessWebsite | login | — | — | -| (empty platformId) | pamMachine | ssh | 22 | -| (unknown platformId) | pamMachine | ssh | 22 | +| Custom/renamed platform resolved by CyberArk metadata or keyword matching | inferred PAM type | inferred | inferred | +| Empty/unknown platform with no metadata or keyword match | login | — | — | + +For an unresolved platform, fields not already represented by the login's title, +login, password, URL, or CyberArk notes are preserved as text custom fields. +Nested account metadata is flattened using dotted field names. Raw secret-bearing +payload keys are excluded from custom fields. ### Field Mapping @@ -170,6 +175,9 @@ pam project cyberark-import pvwa.company.com --dry-run --output import.json --in # Filter specific safes pam project cyberark-import pvwa.company.com --safes "Production,Staging" --exclude-safes "Archive*" +# Import into Nested Share Folders (folders, records, rotation, PAM config) +pam project cyberark-import pvwa.company.com --name "CyberArk Migration" --gateway "My Gateway" --nsf + # Extend existing project pam project cyberark-import pvwa.company.com --config @@ -199,7 +207,8 @@ pam project cyberark-cleanup --name "CyberArk Migration" --dry-run | `--name`, `-n` | Project name | | `--config`, `-c` | Extend existing PAM config UID | | `--gateway`, `-g` | Gateway name or UID | -| `--folder-mode` | flat, exact, ksm (default) | +| `--folder-mode` | flat, exact, ksm, safe (default: safe) | +| `--nsf` | Create project folders/records/PAM config in Nested Share Folders | | `--safes` | Include only these safes (comma/glob) | | `--exclude-safes` | Exclude safes (comma/glob) | | `--list-safes` | List safes and exit | @@ -273,5 +282,3 @@ Before building the import JSON, the importer warns about: - Rate limit handling: automatic retry on HTTP 429 with exponential backoff - Pagination cap: MAX_FETCH_RECORDS (50,000) prevents OOM attacks --- - - diff --git a/keepercommander/__init__.py b/keepercommander/__init__.py index 7752c3999..a67172dfa 100644 --- a/keepercommander/__init__.py +++ b/keepercommander/__init__.py @@ -10,4 +10,4 @@ # Contact: commander@keepersecurity.com # -__version__ = '18.1.5' +__version__ = '18.1.6' diff --git a/keepercommander/__main__.py b/keepercommander/__main__.py index b627a7947..58772abf9 100644 --- a/keepercommander/__main__.py +++ b/keepercommander/__main__.py @@ -201,6 +201,14 @@ def main(from_package=False): if from_package: sys.excepthook = handle_exceptions + # Internal: background service mode for a frozen (PyInstaller) executable - see service_app.py. + is_frozen = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') + if is_frozen: + from .service.core.service_app import SERVICE_MODE_FLAG, run_background_service + if len(sys.argv) > 1 and sys.argv[1] == SERVICE_MODE_FLAG: + run_background_service() + return + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) opts, flags = parser.parse_known_args(sys.argv[1:]) diff --git a/keepercommander/api.py b/keepercommander/api.py index 1e45403f0..4d611e5fb 100644 --- a/keepercommander/api.py +++ b/keepercommander/api.py @@ -43,6 +43,7 @@ from .sync_down import sync_down from .team import Team from .ttk import TTK +from .sanitization import sanitize_protobuf_json current_milli_time = lambda: int(round(time.time() * 1000)) @@ -881,7 +882,8 @@ def communicate_rest(params, request, endpoint, *, rs_type=None, payload_version if request: if logging.getLogger().level <= logging.DEBUG: js = google.protobuf.json_format.MessageToJson(request) - logging.debug('>>> [RQ] %s: %s', endpoint, js) + sanitized_js = sanitize_protobuf_json(js) + logging.debug('>>> [RQ] %s: %s', endpoint, sanitized_js) api_request_payload.payload = request.SerializeToString() if isinstance(payload_version, int): api_request_payload.apiVersion = payload_version @@ -894,7 +896,8 @@ def communicate_rest(params, request, endpoint, *, rs_type=None, payload_version proto_rs.ParseFromString(rs) if logging.getLogger().level <= logging.DEBUG: js = google.protobuf.json_format.MessageToJson(proto_rs) - logging.debug('>>> [RS] %s: %s', endpoint, js) + sanitized_js = sanitize_protobuf_json(js) + logging.debug('>>> [RS] %s: %s', endpoint, sanitized_js) return proto_rs else: return rs diff --git a/keepercommander/commands/discoveryrotation.py b/keepercommander/commands/discoveryrotation.py index 9be7a7711..f8aee46fe 100644 --- a/keepercommander/commands/discoveryrotation.py +++ b/keepercommander/commands/discoveryrotation.py @@ -569,6 +569,25 @@ class PAMCreateRecordRotationCommand(Command): def get_parser(self): return PAMCreateRecordRotationCommand.parser + def execute_args(self, params, args, **kwargs): + # Reached only via normal command dispatch (interactive shell, a one-shot + # `keeper pam rotation edit ...` invocation, or a run-batch script line) - + # never by callers that instantiate this class directly and call execute() + # themselves (bulk PAM import/extend), who keep trusting their own + # pre-warmed record_rotation_cache for speed. + kwargs.setdefault('trust_cache', False) + # Only a literal False opts into the forced-resync path; any other value + # (True, None, or anything else a caller might pass) is treated as True. + trust_cache = kwargs['trust_cache'] is not False + kwargs['trust_cache'] = trust_cache + try: + return super().execute_args(params, args, **kwargs) + finally: + if not trust_cache: + # Make sure the shell picks up the change (or the fresh sync we just + # forced) via the standard "sync after command" mechanism in do_command(). + params.sync_data = True + def execute(self, params, **kwargs): """Configure rotation settings for one or multiple PAM records. @@ -577,6 +596,9 @@ def execute(self, params, **kwargs): resource linkage and then submits rotation requests to the Keeper PAM router service. """ + # Only a literal False opts into the forced-resync path; any other value + # (True, missing/defaulted True, None, or anything else) is treated as True. + trust_cache = kwargs.pop('trust_cache', True) is not False def config_resource(_dag, target_record, target_config_uid, silent=None): if not _dag.linking_dag.has_graph: @@ -1497,6 +1519,15 @@ def config_user(_dag, target_record, target_resource_uid, target_config_uid=None if not kwargs.get('silent'): logging.info('Selected %d PAM record(s) for rotation', len(pam_records)) + if not trust_cache: + # Refresh the local rotation cache before reading any cached revision below. + # A stale rotation revision here would collide with a real rotation change made + # elsewhere (another session, a scheduled rotation) since this cache was + # last warmed, and the router rejects the update with an error like + # mismatched_revision_blocking_update. Bulk callers that pre-warm record_rotation_cache + # themselves (pam project import/extend) pass trust_cache=True and skip this. + api.sync_down(params) + schedule_config = kwargs.get('schedule_config') is True if schedule_config: refresh_vault_for_schedule_config(params) @@ -2429,7 +2460,7 @@ def print_root_rotation_setting(params, is_verbose=False, format_type='table'): for c in configurations: # type: vault.TypedRecord if c.record_type in ('pamAwsConfiguration', 'pamAzureConfiguration', 'pamGcpConfiguration', 'pamDomainConfiguration', 'pamNetworkConfiguration', 'pamOciConfiguration', - 'pamGitHubConfiguration'): + 'pamGitHubConfiguration', 'pamHashiCorpConfiguration'): facade.record = c folder_info = resolve_pam_config_folder_info( params, facade, c.record_uid) @@ -2489,7 +2520,7 @@ def print_root_rotation_setting(params, is_verbose=False, format_type='table'): common_parser = argparse.ArgumentParser(add_help=False) common_parser.add_argument('--environment', '-env', dest='config_type', action='store', - choices=['local', 'aws', 'azure', 'gcp', 'domain', 'oci', 'github'], help='PAM Configuration Type') + choices=['local', 'aws', 'azure', 'gcp', 'domain', 'oci', 'github', 'hashicorp'], help='PAM Configuration Type') common_parser.add_argument('--title', '-t', dest='title', action='store', help='Title of the PAM Configuration') common_parser.add_argument('--gateway', '-g', dest='gateway_uid', action='store', help='Gateway UID or Name') common_parser.add_argument('--shared-folder', '-sf', dest='shared_folder_uid', action='store', @@ -2555,12 +2586,23 @@ def print_root_rotation_setting(params, is_verbose=False, format_type='table'): github_group.add_argument('--github-base-url', dest='github_base_url', action='store', help='GitHub Base URL') +hashicorp_group = common_parser.add_argument_group('hashicorp', 'HashiCorp Vault configuration') +hashicorp_group.add_argument('--hashicorp-id', dest='hashicorp_id', action='store', help='HashiCorp Id') +hashicorp_group.add_argument('--vault-base-url', dest='vault_base_url', action='store', + help='Vault Base URL (e.g., https://vault.company.com:8200)') +hashicorp_group.add_argument('--vault-token', dest='vault_token', action='store', + help='Vault Token (optional; syncIdentity takes precedence)') +hashicorp_group.add_argument('--vault-namespace', dest='vault_namespace', action='store', + help='Vault Namespace (optional; leave blank for Community Edition)') +hashicorp_group.add_argument('--vault-mount-path', dest='vault_mount_path', action='store', + help='Vault KV Mount Path (optional; defaults to "secret")') + class PamConfigurationEditMixin(RecordEditMixin): pam_record_types = None PAM_CONFIG_RECORD_TYPES = frozenset({ 'pamAwsConfiguration', 'pamAzureConfiguration', 'pamGcpConfiguration', 'pamDomainConfiguration', 'pamNetworkConfiguration', 'pamOciConfiguration', - 'pamGitHubConfiguration', + 'pamGitHubConfiguration', 'pamHashiCorpConfiguration', }) PAM_RESOURCE_RECORD_TYPES = frozenset({ 'pamDatabase', 'pamDirectory', 'pamMachine', 'pamRemoteBrowser', @@ -2859,6 +2901,22 @@ def parse_properties(self, params, record, **kwargs): # type: (KeeperParams, va oci_region = kwargs.get('oci_region') if oci_region: extra_properties.append(f'text.regionOci={oci_region}') + elif record.record_type == 'pamHashiCorpConfiguration': + hashicorp_id = kwargs.get('hashicorp_id') + if hashicorp_id: + extra_properties.append(f'text.pamHashiCorpId={hashicorp_id}') + vault_base_url = kwargs.get('vault_base_url') + if vault_base_url: + extra_properties.append(f'text.pamHashiCorpVaultBaseUrl={vault_base_url}') + vault_token = kwargs.get('vault_token') + if vault_token: + extra_properties.append(f'secret.pamHashiCorpVaultToken={vault_token}') + vault_namespace = kwargs.get('vault_namespace') + if vault_namespace: + extra_properties.append(f'text.pamHashiCorpVaultNamespace={vault_namespace}') + vault_mount_path = kwargs.get('vault_mount_path') + if vault_mount_path: + extra_properties.append(f'text.pamHashiCorpVaultMountPath={vault_mount_path}') if extra_properties: self.assign_typed_fields(record, [RecordEditMixin.parse_field(x) for x in extra_properties]) @@ -2936,9 +2994,11 @@ def execute(self, params, **kwargs): record_type = 'pamDomainConfiguration' elif config_type == 'oci': record_type = 'pamOciConfiguration' + elif config_type == 'hashicorp': + record_type = 'pamHashiCorpConfiguration' else: raise CommandError('pam-config-new', f'--environment {config_type} is not supported' - ' - supported options: local, aws, azure, gcp, domain, oci, github') + ' - supported options: local, aws, azure, gcp, domain, oci, github, hashicorp') title = kwargs.get('title') if not title: @@ -3265,7 +3325,34 @@ def execute(self, params, **kwargs): if rri_status_name == 'RRS_ONLINE': configuration_uid = utils.base64_url_encode(rri.configurationUid) - gateway_name = rri.controllerName if rri.controllerName else '-' + gateway_name = rri.controllerName + if not gateway_name and rri.controllerUid: + def _normalize_uid(uid): + if uid is None: + return None + if isinstance(uid, (bytes, bytearray)): + return utils.base64_url_encode(uid) + return str(uid) + + target_uid = _normalize_uid(rri.controllerUid) + if target_uid is None: + gateway_name = None + else: + try: + all_gateways = gateway_helper.get_all_gateways(params) or [] + except (Exception,) as ex: + logging.debug(f"Failed to retrieve gateway list for name resolution: {ex}") + all_gateways = [] + + if all_gateways: + matched = next((g for g in all_gateways + if _normalize_uid(getattr(g, 'controllerUid', None)) == target_uid), None) + if matched: + gateway_name = getattr(matched, 'controllerName', None) + if gateway_name: + logging.debug(f"Resolved gateway name from controllerUid {target_uid} -> {gateway_name}") + + gateway_name = gateway_name if gateway_name else '-' gateway_uid = utils.base64_url_encode(rri.controllerUid) if rri.controllerUid else '-' def is_resource_ok(resource_id, params, configuration_uid): @@ -3754,6 +3841,28 @@ def _is_rotation_allowed_by_enforcement(params): return True +def ensure_gateway_management_allowed(params): + # type: (KeeperParams) -> bool + """Per-user enforcement gate on the 'allow_pam_gateway' role enforcement + (confirmed via live account_summary payload). Prints an error and returns + False when the user's enterprise enforcement disallows Gateway management. + Shared by both PAMCreateGatewayCommand and PAMGatewayRemoveCommand (and + their legacy discoveryrotation_v1 counterparts) to avoid duplicating the + check at every call site. + """ + try: + from .workflow.helpers import is_pam_action_allowed_by_enforcement + except ImportError as _e: + logging.debug('workflow.helpers not available; skipping gateway enforcement check: %s', _e) + return True + + if not is_pam_action_allowed_by_enforcement(params, 'allow_pam_gateway'): + print(f"{bcolors.FAIL}Gateway management is not allowed by your enterprise " + f"enforcement (allow_pam_gateway).{bcolors.ENDC}") + return False + return True + + class PAMGatewayActionRotateCommand(Command): parser = argparse.ArgumentParser(prog='pam action rotate') parser.add_argument('--record-uid', '-r', dest='record_uid', action='store', @@ -4393,6 +4502,9 @@ def get_parser(self): return PAMGatewayRemoveCommand.dr_remove_controller_parser def execute(self, params, **kwargs): + if not ensure_gateway_management_allowed(params): + return + gateway_name = kwargs.get('gateway') gateways = gateway_helper.get_all_gateways(params) @@ -4461,6 +4573,8 @@ def get_parser(self): return PAMCreateGatewayCommand.dr_create_controller_parser def execute(self, params, **kwargs): + if not ensure_gateway_management_allowed(params): + return gateway_name = kwargs.get('gateway_name') ksm_app = kwargs.get('ksm_app') diff --git a/keepercommander/commands/discoveryrotation_v1.py b/keepercommander/commands/discoveryrotation_v1.py index 8d2d21502..1468dcf62 100644 --- a/keepercommander/commands/discoveryrotation_v1.py +++ b/keepercommander/commands/discoveryrotation_v1.py @@ -22,7 +22,7 @@ from keeper_secrets_manager_core.utils import url_safe_str_to_bytes from .base import Command, GroupCommand, user_choice, dump_report_data, report_output_parser, json_output_parser, field_to_title, FolderMixin -from .discoveryrotation import PAMLegacyCommand +from .discoveryrotation import PAMLegacyCommand, ensure_gateway_management_allowed from .folder import FolderMoveCommand from .ksm import KSMCommand from .pam import gateway_helper, router_helper @@ -1710,6 +1710,9 @@ def get_parser(self): return PAMGatewayRemoveCommand.dr_remove_controller_parser def execute(self, params, **kwargs): + if not ensure_gateway_management_allowed(params): + return + gateway_name = kwargs.get('gateway') gateways = gateway_helper.get_all_gateways(params) @@ -1745,6 +1748,8 @@ def get_parser(self): return PAMCreateGatewayCommand.dr_create_controller_parser def execute(self, params, **kwargs): + if not ensure_gateway_management_allowed(params): + return gateway_name = kwargs.get('gateway_name') ksm_app = kwargs.get('ksm_app') diff --git a/keepercommander/commands/enterprise.py b/keepercommander/commands/enterprise.py index 57fffe7a6..dc5784e00 100644 --- a/keepercommander/commands/enterprise.py +++ b/keepercommander/commands/enterprise.py @@ -51,6 +51,13 @@ from ..proto import record_pb2, APIRequest_pb2, enterprise_pb2, automator_pb2, pam_pb2 +# Privileges that require the current user to already hold them before they can be granted to, +# or removed from, another role. These control account transfer, team/company management, and +# financial operations. manage_nodes/sharing_administrator are excluded — restricting those would +# block delegated admins from routine management. +_PRIVILEGED_GRANTS = frozenset(('transfer_account', 'manage_companies', 'manage_teams')) + + def register_commands(commands): commands['enterprise-down'] = GetEnterpriseDataCommand() commands['enterprise-info'] = EnterpriseInfoCommand() @@ -110,6 +117,11 @@ def register_command_info(aliases, command_info): SUPPORTED_TEAM_COLUMNS = ['restricts', 'node', 'user_count', 'users', 'queued_user_count', 'queued_users', 'role_count', 'roles'] SUPPORTED_ROLE_COLUMNS = ['visible_below', 'default_role', 'admin', 'node', 'user_count', 'users', 'team_count', 'teams', 'enforcement_count', 'enforcements', 'managed_node_count', 'managed_nodes', 'managed_nodes_permissions'] +# Always present in the output row/header, regardless of --columns; not selectable so excluded from validation. +BASE_NODE_COLUMNS = {'node_id', 'name'} +BASE_USER_COLUMNS = {'user_id', 'email'} +BASE_TEAM_COLUMNS = {'team_uid', 'name'} +BASE_ROLE_COLUMNS = {'role_id', 'name'} enterprise_data_parser = argparse.ArgumentParser(prog='enterprise-down', description='Download & decrypt enterprise data.') @@ -397,7 +409,8 @@ def execute(self, params, **kwargs): nodes[node_id] = { 'node_id': node_id, 'parent_id': node.get('parent_id') or 0, - 'name': (node['data'].get('displayname') or '') if node.get('parent_id') else params.enterprise['enterprise_name'], + 'name': node['data'].get('displayname') or + (params.enterprise['enterprise_name'] if not node.get('parent_id') else ''), 'isolated': node.get('restrict_visibility') or False, 'users': [], 'teams': [], @@ -650,7 +663,7 @@ def tree_node(node): if len(columns) == 0: columns.update(('parent_node', 'parent_id', 'user_count', 'team_count', 'role_count')) else: - wc = columns.difference(supported_columns) + wc = columns.difference(supported_columns, BASE_NODE_COLUMNS) if len(wc) > 0: logging.warning('\n\nSupported node columns: %s\n', ', '.join(supported_columns)) @@ -755,7 +768,7 @@ def tree_node(node): if len(columns) == 0: columns.update(('name', 'status', 'transfer_status', 'node')) else: - wc = columns.difference(supported_columns) + wc = columns.difference(supported_columns, BASE_USER_COLUMNS) if len(wc) > 0: logging.warning('\n\nSupported user columns: %s\n', ', '.join(supported_columns)) @@ -867,7 +880,7 @@ def tree_node(node): if len(params.enterprise['queued_team_users']) > 0: columns.update(('queued_user_count',)) else: - wc = columns.difference(supported_columns) + wc = columns.difference(supported_columns, BASE_TEAM_COLUMNS) if len(wc) > 0: logging.warning('\n\nSupported team columns: %s\n', ', '.join(supported_columns)) @@ -931,7 +944,7 @@ def tree_node(node): if len(columns) == 0: columns.update(('default_role', 'admin', 'node', 'user_count')) else: - wc = columns.difference(supported_columns) + wc = columns.difference(supported_columns, BASE_ROLE_COLUMNS) if len(wc) > 0: logging.warning('\n\nSupported role columns: %s\n', ', '.join(supported_columns)) @@ -1236,19 +1249,44 @@ def execute(self, params, **kwargs): if not matched_nodes: raise CommandError('enterprise-node', 'No nodes to toggle.') + toggled_nodes = [] for mn in matched_nodes: node_id = mn['node_id'] data = mn['data'] - displayname = data['displayname'] + displayname = data.get('displayname') or str(node_id) + was_isolated = bool(mn.get('restrict_visibility')) + is_root = not mn.get('parent_id') request = enterprise_pb2.SetRestrictVisibilityRequest() - request.nodeId = node_id + # Root isolation is an enterprise-level flag returned in + # GeneralDataEntity rather than on the root Node entity. + request.nodeId = 0 if is_root else node_id try: api.communicate_rest(params, request, 'enterprise/set_restrict_visibility') - mn['restrict_visibility'] = not (mn.get('restrict_visibility') or False) - logging.warning('good result: {}'.format(displayname)) + toggled_nodes.append((node_id, displayname, was_isolated)) except Exception as e: logging.warning('node \"%s\": toggle isolation failed: %s', displayname, e) - api.query_enterprise(params) + if toggled_nodes: + api.query_enterprise(params, force=True) + refreshed_nodes = { + x['node_id']: x for x in (params.enterprise or {}).get('nodes', []) + } + for node_id, displayname, was_isolated in toggled_nodes: + refreshed_node = refreshed_nodes.get(node_id) + if not refreshed_node: + logging.warning( + 'node \"%s\": isolation toggle could not be verified after refresh', + displayname) + continue + is_isolated = bool(refreshed_node.get('restrict_visibility')) + if is_isolated == was_isolated: + logging.warning( + 'node \"%s\": server accepted the isolation toggle, ' + 'but the state did not change', + displayname) + else: + logging.info( + 'node \"%s\": isolation is now %s', + displayname, 'enabled' if is_isolated else 'disabled') else: for node_name in unmatched_nodes: logging.warning('Node \'%s\' is not found: Skipping', node_name) @@ -1554,7 +1592,9 @@ def is_in_chain(node_id, parent_id): 'node_id': node['node_id'], 'encrypted_data': encrypted_data } - rq['parent_id'] = parent_id if parent_id else node.get('parent_id') + effective_parent_id = parent_id if parent_id else node.get('parent_id') + if effective_parent_id: + rq['parent_id'] = effective_parent_id request_batch.append(rq) if request_batch: @@ -2260,6 +2300,72 @@ def is_node_managed_by_role(params, node_id, role_id): # type: (KeeperParams, i managed_nodes = params.enterprise.get('managed_nodes') return any(True for x in managed_nodes if x.get('managed_node_id') == node_id and x.get('role_id') == role_id) + @staticmethod + def get_effective_privileges_for_node(params, role_id, target_node_id): + # type: (KeeperParams, int, int) -> Set[str] + """Compute effective privileges for a role on a target node, honoring cascade. + + Walks up the ancestor chain to find privileges granted on parent nodes + that cascade down to the target node. + """ + effective_privs = set() # type: Set[str] + + # Build node lookup and parent map + nodes = {n['node_id']: n for n in params.enterprise.get('nodes', [])} + if target_node_id not in nodes: + return effective_privs + + # Build managed_nodes by role_id with cascade info + role_managed = {} # type: Dict[int, Dict[str, Any]] + for mn in params.enterprise.get('managed_nodes', []): + if mn['role_id'] == role_id: + role_managed[mn['managed_node_id']] = { + 'cascade': mn.get('cascade_node_management', False) + } + + # Walk up from target_node_id to root, collecting managed ancestor nodes + current_node_id = target_node_id + managed_ancestors = [] # type: List[int] + + while current_node_id: + if current_node_id in role_managed: + managed_ancestors.append(current_node_id) + node = nodes.get(current_node_id) + if node and node.get('parent_id'): + current_node_id = node['parent_id'] + else: + current_node_id = None + + # Collect privileges from managed ancestors that cascade or are exact matches + for rp in params.enterprise.get('role_privileges', []): + if rp['role_id'] != role_id: + continue + privilege = rp['privilege'].lower() + managed_node_id = rp['managed_node_id'] + + if managed_node_id == target_node_id: + # Exact match always applies + effective_privs.add(privilege) + elif managed_node_id in managed_ancestors: + # Ancestor privilege applies if cascade is enabled + if role_managed[managed_node_id]['cascade']: + effective_privs.add(privilege) + + return effective_privs + + @staticmethod + def get_current_enterprise_user_id(params): + # type: (KeeperParams) -> Optional[int] + """Find the current user's enterprise_user_id by username matching.""" + if 'users' not in params.enterprise: + return None + + username_lower = params.user.lower() if params.user else None + for user in params.enterprise['users']: + if user.get('username', '').lower() == username_lower: + return user.get('enterprise_user_id') + return None + def execute(self, params, **kwargs): if kwargs.get('add') and kwargs.get('remove'): raise CommandError('enterprise-role', "'add' and 'delete' parameters are mutually exclusive.") @@ -2655,6 +2761,19 @@ def execute(self, params, **kwargs): else: enforcement_value = None + # Restrict require_account_share enforcement to root admins only. + # Reuses EnterpriseCommand.get_user_root_nodes(), which fails open (treats an + # unresolvable current user as main admin) to match the rest of the class. + if key == 'require_account_share': + true_root_node_id = next((n['node_id'] for n in params.enterprise.get('nodes', []) + if n['node_id'] & 0xffffffff == 2), None) + is_root_admin = true_root_node_id is not None and \ + true_root_node_id in self.get_user_root_nodes(params) + if not is_root_admin: + logging.warning('Failed to modify enforcement \'%s\': Only enterprise root administrators can manage account transfer policies', + key) + continue + role_enforcements = params.enterprise.get('role_enforcements') or [] for role in matched_roles: role_id = role['role_id'] @@ -2776,9 +2895,23 @@ def execute(self, params, **kwargs): if not node: logging.warning('Role "%d" does not manage node "%d"', role_id, node_id) return - privileges = {x['privilege'] for x in params.enterprise.get('role_privileges', []) + privileges = {x['privilege'].lower() for x in params.enterprise.get('role_privileges', []) if x['role_id'] == role_id and x['managed_node_id'] == node_id} all_privileges = {x[1].lower() for x in constants.ROLE_PRIVILEGES} + + # Get current user's effective privileges for authorization check. + # Cascade-aware: walks ancestor chain to find privileges granted on parent nodes. + # Fails closed: an unresolvable current user yields no privileges, so the request is denied. + current_user_id = self.get_current_enterprise_user_id(params) + current_user_effective_privileges = set() # type: Set[str] + if current_user_id: + current_user_roles = {x['role_id'] for x in params.enterprise.get('role_users', []) + if x.get('enterprise_user_id') == current_user_id} + for user_role_id in current_user_roles: + # For each role the user has, get effective privileges on this node (honoring cascade) + priv_set = self.get_effective_privileges_for_node(params, user_role_id, node_id) + current_user_effective_privileges.update(priv_set) + for is_add in [True, False]: parameter = 'add_privilege' if is_add else 'remove_privilege' privilege_list = kwargs.get(parameter) @@ -2787,11 +2920,7 @@ def execute(self, params, **kwargs): privilege = privilege.lower() if privilege not in all_privileges: logging.warning('Add/Remove managed node privilege: invalid privilege: %s', privilege) - return - # if is_add: - # if privilege in ['transfer_account', 'manage_companies']: - # logging.warning('Add managed node privilege: Commander does not support \"%s\" privilege', privilege) - # return + continue if is_add and privilege in privileges: logging.info('Add privilege: Role "%d", Mode "%s" already contains privilege "%s" ', role_id, node_id, privilege) @@ -2801,6 +2930,17 @@ def execute(self, params, **kwargs): role_id, node_id, privilege) continue + # Verify caller holds the privilege before granting or revoking it on another + # role. Gating removal too, since a node admin without the privilege could + # otherwise strip transfer_account/manage_teams/manage_companies from a role + # that depends on it (e.g. an offboarding or security-owning role). + if privilege in _PRIVILEGED_GRANTS: + if privilege not in current_user_effective_privileges: + verb = 'assign' if is_add else 'remove' + logging.warning('Failed to %s \'%s\' privilege: You do not have the required privilege to modify \'%s\'', + verb, privilege, privilege) + continue + rq = { 'command': 'managed_node_privilege_add' if is_add else 'managed_node_privilege_remove', 'role_id': role_id, diff --git a/keepercommander/commands/enterprise_common.py b/keepercommander/commands/enterprise_common.py index eb0bba7e5..f967c93b6 100644 --- a/keepercommander/commands/enterprise_common.py +++ b/keepercommander/commands/enterprise_common.py @@ -27,7 +27,7 @@ def __init__(self): self.public_keys = {} self.team_keys = {} self._node_map = None - self._node_map_enterprise_name = None + self._node_map_signature = None def execute_args(self, params, args, **kwargs): if params.enterprise: @@ -296,11 +296,21 @@ def get_enterprise_ids(params, num_ids=1): return enterprise_ids def get_node_path(self, params, node_id, omit_root=False): - current_enterprise_name = params.enterprise.get('enterprise_name') - if self._node_map is None or self._node_map_enterprise_name != current_enterprise_name: - self._node_map_enterprise_name = current_enterprise_name + node_map_signature = ( + params.enterprise.get('enterprise_name'), + tuple( + (x['node_id'], x.get('parent_id', 0), x['data'].get('displayname'), x.get('name')) + for x in params.enterprise['nodes'] + ), + ) + if self._node_map is None or self._node_map_signature != node_map_signature: + self._node_map_signature = node_map_signature self._node_map = { - x['node_id']: (x['data'].get('displayname') or x.get('name') or str(x['node_id']) if x.get('parent_id', 0) > 0 else params.enterprise['enterprise_name'], x.get('parent_id', 0)) + x['node_id']: ( + x['data'].get('displayname') or x.get('name') or + (params.enterprise['enterprise_name'] if not x.get('parent_id') else str(x['node_id'])), + x.get('parent_id', 0), + ) for x in params.enterprise['nodes']} path = '' node = self._node_map.get(node_id) @@ -328,11 +338,10 @@ def resolve_nodes(params, name): # type: (KeeperParams, str) -> collections.It yield node continue if node_name: - if 'parent_id' in node: - display_name = node['data'].get('displayname') or '' - else: - display_name = params.enterprise['enterprise_name'] or '' - if display_name and display_name.lower() == node_name: + display_names = [node['data'].get('displayname') or ''] + if not node.get('parent_id'): + display_names.append(params.enterprise['enterprise_name'] or '') + if any(x and x.lower() == node_name for x in display_names): yield node else: if 'parent_id' not in node: diff --git a/keepercommander/commands/pam/vault_target.py b/keepercommander/commands/pam/vault_target.py index adfc63342..71c9c1fcf 100644 --- a/keepercommander/commands/pam/vault_target.py +++ b/keepercommander/commands/pam/vault_target.py @@ -325,6 +325,14 @@ def resolve_pam_record(params, identifier, rec_type=None): if not identifier: return None + try: + from ..pam_import.record_loader import load_pam_record + rec = load_pam_record(params, identifier) + if rec and _record_matches_type(rec, rec_type): + return rec + except Exception: + pass + if identifier in getattr(params, 'record_cache', {}): rec = vault.KeeperRecord.load(params, identifier) if rec and _record_matches_type(rec, rec_type): @@ -619,8 +627,13 @@ def update_pam_record(params, record, command='pam', force_nsf=False) -> bool: return False -def execute_record_add_in_folder(params, args, folder_uid, command='pam'): - """Add a record in *folder_uid*, using NSF-native creation when needed.""" +def execute_record_add_in_folder(params, args, folder_uid, command='pam', + sync_after=True): + """Add a record in *folder_uid*, using NSF-native creation when needed. + + When *sync_after* is False, NSF callers can defer sync_down to a batch + boundary (avoids one sync per record during large PAM imports). + """ from ..record_edit import RecordAddCommand from ..nested_share_folder.record_commands import NestedShareRecordAddCommand @@ -630,7 +643,7 @@ def execute_record_add_in_folder(params, args, folder_uid, command='pam'): nsf_args.pop('folder', None) nsf_args['folder_uid'] = folder_uid uid = NestedShareRecordAddCommand().execute(params, **nsf_args) - if uid: + if uid and sync_after: from ..pam_import.nsf_helpers import sync_down_preserving_nsf_keys sync_down_preserving_nsf_keys(params) return uid @@ -639,8 +652,13 @@ def execute_record_add_in_folder(params, args, folder_uid, command='pam'): return RecordAddCommand().execute(params, **record_args) -def execute_record_v3_add_in_folder(params, args, folder_uid, command='pam'): - """Add a v3 typed record in *folder_uid*, using NSF-native creation when needed.""" +def execute_record_v3_add_in_folder(params, args, folder_uid, command='pam', + sync_after=True): + """Add a v3 typed record in *folder_uid*, using NSF-native creation when needed. + + When *sync_after* is False, NSF callers can defer sync_down to a batch + boundary (avoids one sync per record during large PAM imports). + """ import json from ..recordv3 import RecordAddCommand @@ -663,8 +681,9 @@ def execute_record_v3_add_in_folder(params, args, folder_uid, command='pam'): if not result.get('success'): raise CommandError(command, normalize_nsf_user_message(result.get('message')) or 'Failed to create record in Nested Share Folder') - from ..pam_import.nsf_helpers import sync_down_preserving_nsf_keys - sync_down_preserving_nsf_keys(params) + if sync_after: + from ..pam_import.nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) return result['record_uid'] record_args['folder'] = folder_uid diff --git a/keepercommander/commands/pam_debug/gateway.py b/keepercommander/commands/pam_debug/gateway.py index bd6999000..2991d8ee7 100644 --- a/keepercommander/commands/pam_debug/gateway.py +++ b/keepercommander/commands/pam_debug/gateway.py @@ -85,7 +85,7 @@ def execute(self, params: KeeperParams, **kwargs): if record_link.dag.has_graph is True: print(self._h("Record Linking Graph")) graph.do_list(params=params, gateway_context=gateway_context, graph_type="rl", debug_level=debug_level, - indent=1) + indent=1, show_data=False) print("") print(self._h("User to Service/Task Graph")) diff --git a/keepercommander/commands/pam_debug/graph.py b/keepercommander/commands/pam_debug/graph.py index 9b6e56f33..2a08d8c92 100644 --- a/keepercommander/commands/pam_debug/graph.py +++ b/keepercommander/commands/pam_debug/graph.py @@ -1,4 +1,7 @@ from __future__ import annotations +import base64 +import binascii +import json from . import get_connection import argparse import logging @@ -11,13 +14,14 @@ from ...discovery_common.constants import (PAM_USER, PAM_DIRECTORY, PAM_MACHINE, PAM_DATABASE, VERTICES_SORT_MAP, DIS_INFRA_GRAPH_ID, RECORD_LINK_GRAPH_ID, DIS_JOBS_GRAPH_ID) from ...discovery_common.types import (DiscoveryObject, DiscoveryUser, DiscoveryDirectory, DiscoveryMachine, - DiscoveryDatabase, JobContent, ServiceAcl) + DiscoveryDatabase, JobContent, ServiceAcl, UserAcl, UserAclServiceNames) from ...discovery_common.dag_sort import sort_infra_vertices from ...keeper_dag import DAG from ...keeper_dag.types import GRAPH_ID_TO_ENDPOINT, PamEndpoints from ...keeper_dag.vertex import DAGVertex -from ...keeper_dag.edge import EdgeType -from typing import TYPE_CHECKING +from ...keeper_dag.edge import EdgeType, DAGEdge +from ...crypto import decrypt_aes_v2 +from typing import TYPE_CHECKING, Optional, List if TYPE_CHECKING: from ...params import KeeperParams @@ -53,6 +57,8 @@ class PAMDebugGraphCommand(PAMGatewayActionDiscoverCommandBase): dest='format', default="dot", action='store', help='The format of the graph.') parser.add_argument('--debug-gs-level', required=False, dest='debug_level', action='store', help='GraphSync debug level. Default is 0', type=int, default=0) + parser.add_argument('--history-level', required=False, dest='history_level', action='store', + help='Levels of history to keep. Default is 0', type=int, default=0) mapping = { PAM_USER: {"order": 1, "sort": "_sort_name", "item": DiscoveryUser, "key": "user"}, @@ -70,8 +76,25 @@ class PAMDebugGraphCommand(PAMGatewayActionDiscoverCommandBase): def get_parser(self): return PAMDebugGraphCommand.parser + @staticmethod + def is_base64(value) -> bool: + # Ensure the input is treated as bytes (or an ASCII string) + if isinstance(value, str): + value_bytes = value.encode('utf-8') + elif isinstance(value, (bytes, bytearray)): + value_bytes = value + else: + return False + + try: + # validate=True ensures characters outside the base64 alphabet throw an error + base64.b64decode(value_bytes, validate=True) + return True + except binascii.Error: + return False + def _do_text_list_infra(self, params: KeeperParams, gateway_context: GatewayContext, debug_level: int = 0, - indent: int = 0): + indent: int = 0, **kwargs): infra = Infrastructure(record=gateway_context.configuration, params=params, logger=logging, debug_level=debug_level, use_per_graph_endpoints=False) @@ -148,8 +171,80 @@ def _handle(current_vertex: DAGVertex, indent: int = 0, last_record_type: str | _handle(configuration, indent=indent) print("") + def _do_acl(self, + acl_edge:DAGEdge, + record_key: bytes, + params: KeeperParams, + pad: Optional[str] = ""): + + acl = acl_edge.content_as_object(UserAcl) + if acl is None: + print(f"{pad} {self._f('missing ACL')}") + else: + print(f"{pad} . acl: {acl_edge.content}") + print(f"{pad} . path {acl_edge.path}") + if acl.is_iam_user: + print(f"{pad} . is IAM user") + if acl.is_admin: + print(f"{pad} . is the {self._b('Admin')}") + if acl.belongs_to: + print(f"{pad} . local user") + else: + print(f"{pad} . looks like directory user") + + if acl.controls_services: + print(f"{pad} . controls services") + + if acl.service_names: + + for service_names in acl.get_service_names(record_key): # type: UserAclServiceNames + print(f"{pad} - {service_names.type.value}") + for item in service_names.items: + print(f"{pad} > {item.name} " + f"{'... via discovery' if item.via_discovery else '... manual'}") + else: + print(f"{pad} . service names not set") + else: + print(f"{pad} . does not control services") + + if acl.rotation_settings is not None: + if acl.rotation_settings.schedule: + print(f"{pad} . schedule: {acl.rotation_settings.get_schedule()}") + else: + print(f"{pad} . schedule: None") + if acl.rotation_settings.pwd_complexity: + print(f"{pad} . password complexity: {acl.rotation_settings.get_pwd_complexity(record_key)}") + else: + print(f"{pad} . password complexity: None") + if acl.rotation_settings.noop: + print(f"{pad} . is a NOOP") + if acl.rotation_settings.disabled: + print(f"{pad} . rotation is disabled") + + if (acl.rotation_settings.saas_record_uid_list is not None + and len(acl.rotation_settings.saas_record_uid_list) > 0): + print(f"{pad} . has SaaS rotation: " + f"{acl.rotation_settings.saas_record_uid_list[0]}") + + if len(acl.rotation_settings.saas_record_uid_list) > 0: + if acl.rotation_settings.noop: + saas_config_uid = acl.rotation_settings.saas_record_uid_list[0] + saas_config = load_pam_record( + params, + saas_config_uid) # type: TypedRecord | None + + print(f" . SaaS configuration record is {saas_config.title}") + else: + print(f"{bcolors.FAIL} . Has SaaS plugin config record, " + f"however it's not NOOP{bcolors.ENDC}") + else: + print(f"{pad}{bcolors.FAIL} . there are no rotation settings in ACL!{bcolors.ENDC}") + def _do_text_list_rl(self, params: KeeperParams, gateway_context: GatewayContext, debug_level: int = 0, - indent: int = 0): + indent: int = 0, + record_uids: Optional[List[str]] = None, + show_data: bool = True, + **kwargs): print("") @@ -163,12 +258,12 @@ def _do_text_list_rl(self, params: KeeperParams, gateway_context: GatewayContext debug_level=debug_level, use_per_graph_endpoints=False) configuration = record_link.dag.get_root - record = load_pam_record(params, configuration.uid) # type: TypedRecord | None - if record is None: + config_record = load_pam_record(params, configuration.uid) # type: TypedRecord | None + if config_record is None: print(self._f("Configuration record does not exists.")) return - print(self._h(f"{pad}{record.record_type}, {record.title}, {record.record_uid}")) + print(self._h(f"{pad}{config_record.record_type}, {config_record.title}, {config_record.record_uid}")) if configuration.has_data: try: @@ -191,6 +286,24 @@ def _group(configuration_vertex: DAGVertex) -> dict: } for vertex in configuration_vertex.has_vertices(): + + users_uids: Optional[List] = None + if record_uids is not None: + found = vertex.uid in record_uids + if not found: + # Get the resource's users + children_uids = [x.uid for x in vertex.has_vertices() if x is not None] + for record_uid in record_uids: + if record_uid in children_uids: + found = True + if users_uids is None: + users_uids = [] + users_uids.append(record_uid) + break + + if not found: + continue + record = load_pam_record(params, vertex.uid) # type: TypedRecord | None if record is None: group[PAMDebugGraphCommand.NO_RECORD].append({ @@ -202,95 +315,142 @@ def _group(configuration_vertex: DAGVertex) -> dict: rt = PAMDebugGraphCommand.OTHER group[rt].append({ "v": vertex, - "r": record + "r": record, + "u": users_uids }) return group group = _group(configuration) - + for record_type in [PAM_USER, PAM_DIRECTORY, PAM_MACHINE, PAM_DATABASE]: if len(group[record_type]) > 0: print(f"{pad} " + self._b(self._n(record_type))) for item in group[record_type]: vertex = item.get("v") # type: DAGVertex record = item.get("r") # type: TypedRecord + user_uids = item.get("u") # type: Optional[List[str]] text = self._gr(f"{record.title}; {record.record_uid} ") if not vertex.active: text += " " + self._f("Inactive") print(f"{pad} * {text}") - # These are cloud users - if record_type == PAM_USER: - acl = record_link.get_acl(vertex.uid, configuration.uid) - if acl is None: - print(f"{pad} {self._f('missing ACL')}") - else: - if acl.is_iam_user: - print(f"{pad} . is IAM user") - if acl.is_admin: - print(f"{pad} . is the {self._b('Admin')}") - if acl.belongs_to: - print(f"{pad} . belongs to this resource") - else: - print(f"{pad} . looks like directory user") + # Resource connection to configuration. - if acl.rotation_settings: - if acl.rotation_settings.noop: - print(f"{pad} . is a NOOP") - if acl.rotation_settings.disabled: - print(f"{pad} . rotation is disabled") + for edge in vertex.edges: + if not edge: + continue - if (acl.rotation_settings.saas_record_uid_list is not None - and len(acl.rotation_settings.saas_record_uid_list) > 0): - print(f"{pad} . has SaaS rotation: " - f"{acl.rotation_settings.saas_record_uid_list[0]}") + print(f"{pad} {bcolors.WARNING}{edge.edge_type.value.upper()}{bcolors.ENDC}") + print(f"{pad} . path = {edge.path}") + print(f"{pad} . active = {edge.active}") - continue + if edge.edge_type == EdgeType.ACL: + self._do_acl(acl_edge=edge, + params=params, + pad=pad, + record_key=record.record_key) + else: + if show_data and edge.content and edge.content is not None: + try: + data = edge.content_as_dict + if data is not None: + print(f"{pad} . data") + for k, v in data.items(): + print(f"{pad} + {k} = {v}") + else: + print(f"{pad} . data is None") + except Exception as err: + content = edge.content + print(f"{pad} ! data not JSON: {err}") + print(f"{pad} {content}") - if vertex.has_data: - try: - data = vertex.content_as_dict - print(f"{pad} . data") - for k, v in data.items(): - print(f"{pad} + {k} = {v}") - except Exception as err: - print(f"{pad} ! data not JSON: {err}") + if self.is_base64(content): + print(f"{pad} . is base64") + content = base64.b64decode(content) + try: + content = json.loads(content) + print(f"{pad} . is JSON") + for k, v in content.items(): + print(f"{pad} + {k} = {v}") + except (Exception): + + try: + content = decrypt_aes_v2(content, record.record_key) + print(f"{pad} . encrypted with resource record bytes") + except (Exception,): + try: + content = decrypt_aes_v2(content, config_record.record_key) + print(f"{pad} . encrypted with configuration record bytes") + except (Exception,): + print(f"{pad} !! cannot decrypt") + content = None + + if content is not None: + try: + content = json.loads(content) + print(f"{pad} . is JSON") + for k, v in content.items(): + print(f"{pad} + {k} = {v}") + except (Exception): + print(f"{pad} !! decrypt data is not JSON") + + # Get the resource's users children = vertex.has_vertices() if len(children) > 0: + + print("") + print(f"{pad} {bcolors.BOLD}Users{bcolors.ENDC}") + bad = [] for child in children: + if user_uids is not None and child.uid not in user_uids: + continue + child_record = load_pam_record(params, child.uid) # type: TypedRecord | None if child_record is None: if child.active: bad.append(self._f(f"- Record UID {child.uid} does not exists.")) continue else: - print(f"{pad} - {child_record.title}; {child_record.record_uid}") - acl = record_link.get_acl(child.uid, vertex.uid) - if acl is None: - print(f"{pad} {self._f('missing ACL')}") - else: - if acl.is_admin: - print(f"{pad} . is the {self._b('Admin')}") - if acl.belongs_to: - print(f"{pad} . belongs to this resource") + print(f"{pad} * {bcolors.OKBLUE}{child_record.title}{bcolors.ENDC}; " + f"{child_record.record_uid}") + + for edge in child.edges: + if edge and edge.head_uid == child.uid: + if edge.content: + try: + data = edge.content_as_dict + if data is not None: + print(f"{pad} . data") + for k, v in data.items(): + print(f"{pad} + {k} = {v}") + else: + print(f"{pad} . data is None") + except Exception as err: + content = edge.content + print(f"{pad} ! data not JSON: {err}") + print(f"{pad} {content}") + + for edge in child.edges: + if edge is None or edge.head_uid != record.record_uid: + continue + + if edge.edge_type == EdgeType.ACL: + self._do_acl(acl_edge=edge, + params=params, + pad=pad, + record_key=child_record.record_key) else: - print(f"{pad} . looks like directory user") - - if child.has_data: - try: - data = child.content_as_dict - print(f"{pad} . data") - for k, v in data.items(): - print(f"{pad} + {k} = {v}") - except Exception as err: - print(f"{pad} ! data not JSON: {err}") + print(f"{pad} {self._f('not ACL')}") for i in bad: - print(f"{pad} " + i) + print(f"{pad} " + i) + + print("") if len(group[PAMDebugGraphCommand.OTHER]) > 0: + print("") print(f"{pad} " + self._b("Other PAM Types")) for item in group[PAMDebugGraphCommand.OTHER]: vertex = item.get("v") # type: DAGVertex @@ -301,15 +461,37 @@ def _group(configuration_vertex: DAGVertex) -> dict: print(f"{pad} * {text}") if len(group[PAMDebugGraphCommand.NO_RECORD]) > 0: - - # TODO: Check the infra graph for information - print(f"{pad} " + self._b(self._n("In Graph, No Vault Record"))) + print("") + print(f"{pad} " + self._b("No Record; In Graph; Active")) for item in group[PAMDebugGraphCommand.NO_RECORD]: vertex = item.get("v") # type: DAGVertex - print(f"{pad} * {vertex.uid}") + if not vertex.active: + continue + for edge in vertex.edges: + if edge is None or edge.content is None or not edge.active: + continue + text = vertex.uid + if edge.path is not None and edge.path != "": + text += f"; path: {edge.path}" + if not vertex.active: + text += " " + self._f("Inactive") + print(f"{pad} * {text}") + + if show_data and edge.content: + try: + data = edge.content_as_dict + if data is not None: + print(f"{pad} . data") + for k, v in data.items(): + print(f"{pad} + {k} = {v}") + else: + print(f"{pad} . data is None") + except Exception as err: + print(f"{pad} ! data not JSON") + print(f"{pad} {edge.content}") def _do_text_list_service(self, params: KeeperParams, gateway_context: GatewayContext, debug_level: int = 0, - indent: int = 0): + indent: int = 0, **kwargs): pad = "" if indent > 0: @@ -413,7 +595,7 @@ def _do_text_list_service(self, params: KeeperParams, gateway_context: GatewayCo print(f"{pad} * {user}") def _do_text_list_jobs(self, params: KeeperParams, gateway_context: GatewayContext, debug_level: int = 0, - indent: int = 0): + indent: int = 0, **kwargs): infra = Infrastructure(record=gateway_context.configuration, params=params, logger=logging, debug_level=debug_level, fail_on_corrupt=False, use_per_graph_endpoints=False) @@ -578,7 +760,7 @@ def _do_render_jobs(self, params: KeeperParams, gateway_context: GatewayContext, print("") def _do_raw_text_list(self, params: KeeperParams, gateway_context: GatewayContext, graph_id: int = 0, - debug_level: int = 0): + debug_level: int = 0, history_level: int = 0, **kwargs): logging.debug(f"loading graph id {graph_id}, for record uid {gateway_context.configuration.record_uid}") @@ -586,7 +768,8 @@ def _do_raw_text_list(self, params: KeeperParams, gateway_context: GatewayContex endpoint = GRAPH_ID_TO_ENDPOINT[graph_id] dag = DAG(conn=conn, record=gateway_context.configuration, read_endpoint=endpoint, write_endpoint=endpoint, - fail_on_corrupt=False, logger=logging, debug_level=debug_level) + fail_on_corrupt=False, logger=logging, debug_level=debug_level, + history_level=history_level) dag.load(sync_point=0) print("") if dag.is_corrupt is True: @@ -655,13 +838,14 @@ def _handle(current_vertex: DAGVertex, last_vertex: DAGVertex | None = None, ind print("") def _do_raw_render_graph(self, params: KeeperParams, gateway_context: GatewayContext, filepath: str, - graph_format: str, graph_id: int = 0, debug_level: int = 0): + graph_format: str, graph_id: int = 0, debug_level: int = 0, history_level: int = 0): conn = get_connection(params=params) endpoint = GRAPH_ID_TO_ENDPOINT[graph_id] dag = DAG(conn=conn, record=gateway_context.configuration, read_endpoint=endpoint, write_endpoint=endpoint, - fail_on_corrupt=False, logger=logging, debug_level=debug_level) + fail_on_corrupt=False, logger=logging, debug_level=debug_level, + history_level=history_level) dag.load(sync_point=0) dot = dag.to_dot(graph_format=graph_format) if graph_format == "raw": @@ -677,12 +861,13 @@ def _do_raw_render_graph(self, params: KeeperParams, gateway_context: GatewayCon print("") def do_list(self, params: KeeperParams, gateway_context: GatewayContext, graph_type: str, debug_level: int = 0, - indent: int = 0): + indent: int = 0, **kwargs): list_func = getattr(self, f"_do_text_list_{graph_type}") list_func(params=params, gateway_context=gateway_context, debug_level=debug_level, - indent=indent) + indent=indent, + **kwargs) def execute(self, params: KeeperParams, **kwargs): @@ -692,6 +877,7 @@ def execute(self, params: KeeperParams, **kwargs): do_text_list = kwargs.get("do_text_list") do_render = kwargs.get("do_render") debug_level = int(kwargs.get("debug_level", 0)) + history_level = int(kwargs.get("history_level", 0)) configuration_uid = kwargs.get('configuration_uid') try: @@ -710,7 +896,8 @@ def execute(self, params: KeeperParams, **kwargs): self._do_raw_text_list(params=params, gateway_context=gateway_context, graph_id=PAMDebugGraphCommand.graph_id_map.get(graph_type), - debug_level=debug_level) + debug_level=debug_level, + history_level=history_level) if do_render: filepath = kwargs.get("filepath") graph_format = kwargs.get("format") @@ -719,7 +906,8 @@ def execute(self, params: KeeperParams, **kwargs): filepath=filepath, graph_format=graph_format, graph_id=PAMDebugGraphCommand.graph_id_map.get(graph_type), - debug_level=debug_level) + debug_level=debug_level, + history_level=history_level) else: if do_text_list: self.do_list( diff --git a/keepercommander/commands/pam_debug/info.py b/keepercommander/commands/pam_debug/info.py index 74ad5d3db..5e2918c55 100644 --- a/keepercommander/commands/pam_debug/info.py +++ b/keepercommander/commands/pam_debug/info.py @@ -8,6 +8,7 @@ from ...discovery_common.types import UserAcl, DiscoveryObject, ServiceEnum from ...discovery_common.constants import PAM_USER, PAM_MACHINE, PAM_DATABASE, PAM_DIRECTORY from ...keeper_dag import EdgeType +from .graph import PAMDebugGraphCommand import time import re import json @@ -170,117 +171,10 @@ def _print_field(f): record_vertex = record_link.dag.get_vertex(record.record_uid) if record_vertex is not None: - print(self._h("Record Linking")) - - print(self._b(" Record Data (meta)")) - print(f" Raw JSON: {json.dumps(record_vertex.content_as_dict)}") - print("") - - record_parent_vertices = record_vertex.belongs_to_vertices() - print(self._b(" Parent Records")) - if len(record_parent_vertices) > 0: - for record_parent_vertex in record_parent_vertices: - - parent_record = load_pam_record(params, - record_parent_vertex.uid) # type: TypedRecord | None - if parent_record is None: - print(f"{bcolors.FAIL} * Parent record {record_parent_vertex.uid} " - f"does not exists.{bcolors.ENDC}") - continue - - acl_edge = record_vertex.get_edge(record_parent_vertex, EdgeType.ACL) - if acl_edge is not None: - acl_content = acl_edge.content_as_object(UserAcl) # type: UserAcl - print(f" * ACL to {self._n(parent_record.record_type)}; {parent_record.title}; " - f"{record_parent_vertex.uid}") - print(f" . Raw JSON: {json.dumps(acl_edge.content_as_dict)}") - if acl_content.is_admin: - print(f" . Is {self._gr('Admin')}") - if acl_content.belongs_to: - print(f" . Belongs") - else: - print(f" . Is {self._bl('Remote user')}") - - if acl_content.rotation_settings is None: - print(f"{bcolors.FAIL} . There are no rotation settings!{bcolors.ENDC}") - else: - if (acl_content.rotation_settings.schedule is None - or acl_content.rotation_settings.schedule == ""): - print(f" . No Schedule") - else: - print(f" . Schedule = {acl_content.rotation_settings.get_schedule()}") - - if (acl_content.rotation_settings.pwd_complexity is None - or acl_content.rotation_settings.pwd_complexity == ""): - print(f" . No Password Complexity") - else: - key_bytes = record.record_key - print(f" . Password Complexity = " - f"{acl_content.rotation_settings.get_pwd_complexity(key_bytes)}") - print(f" . Disabled = {acl_content.rotation_settings.disabled}") - print(f" . NOOP = {acl_content.rotation_settings.noop}") - print(f" . SaaS configuration record UID = " - f"{acl_content.rotation_settings.saas_record_uid_list}") - - if len(acl_content.rotation_settings.saas_record_uid_list) > 0: - if acl_content.rotation_settings.noop: - saas_config_uid = acl_content.rotation_settings.saas_record_uid_list[0] - saas_config = load_pam_record( - params, - saas_config_uid) # type: TypedRecord | None - - print(f" . SaaS configuration record is {saas_config.title}") - else: - print(f"{bcolors.FAIL} . Has SaaS plugin config record, " - f"however it's not NOOP{bcolors.ENDC}") - - elif record.record_type == PAM_USER: - print(f"{bcolors.FAIL} * PAM User has NO acl!!!!!!{bcolors.ENDC}") - - link_edge = record_vertex.get_edge(record_parent_vertex, EdgeType.LINK) - if link_edge is not None: - print(f" * LINK to {self._n(parent_record.record_type)}; {parent_record.title}; " - f"{record_parent_vertex.uid}") - else: - # This really should not happen - print(f"{bcolors.FAIL} Record does not have a parent record.{bcolors.ENDC}") - print("") - - record_child_vertices = record_vertex.has_vertices() - print(self._b(" Child Records")) - if len(record_child_vertices) > 0: - for record_child_vertex in record_child_vertices: - child_record = load_pam_record(params, - record_child_vertex.uid) # type: TypedRecord | None - - if child_record is None: - print(f"{bcolors.FAIL} * Child record {record_child_vertex.uid} " - f"does not exists.{bcolors.ENDC}") - continue - - acl_edge = record_child_vertex.get_edge(record_vertex, EdgeType.ACL) - link_edge = record_child_vertex.get_edge(record_vertex, EdgeType.LINK) - if acl_edge is not None: - acl_content = acl_edge.content_as_object(UserAcl) - print(f" * ACL from {self._n(child_record.record_type)}; {child_record.title}; " - f"{record_child_vertex.uid}") - if acl_content.is_admin: - print(f" . Is {self._gr('Admin')}") - if acl_content.belongs_to: - print(f" . Belongs") - else: - print(f" . Is {self._bl('Remote user')}") - elif link_edge is not None: - print(f" * LINK from {self._n(child_record.record_type)}; {child_record.title}; " - "{record_child_vertex.uid}") - else: - for edge in record_vertex.edges: # List[DAGEdge] - print(f" * {self._f(edge.edge_type)}?") - - else: - # This is OK - print(f" Record does not have any children.") - print("") + print(self._h("Record Linking Graph")) + graph = PAMDebugGraphCommand() + graph.do_list(params=params, gateway_context=gateway_context, graph_type="rl", debug_level=0, + indent=1, show_data=True, record_uids=[record.record_uid]) else: print(f"{bcolors.FAIL}Cannot find record in record linking.{bcolors.ENDC}") diff --git a/keepercommander/commands/pam_import/base.py b/keepercommander/commands/pam_import/base.py index 85cc852b4..d83db0885 100644 --- a/keepercommander/commands/pam_import/base.py +++ b/keepercommander/commands/pam_import/base.py @@ -982,6 +982,49 @@ def to_dag_dict(self, user_id: str) -> Optional[Dict[str, Any]]: } +def load_custom_text_fields(data) -> list: + """Load import JSON custom text fields for record-add field syntax.""" + if not isinstance(data, list): + return [] + fields = [] + for item in data: + if not isinstance(item, dict): + continue + field_type = str(item.get("type") or "text") + label = str(item.get("label") or "").strip() + raw_value = item.get("value") + if isinstance(raw_value, list): + value = raw_value[0] if raw_value else "" + else: + value = raw_value + if value in (None, ""): + continue + fields.append({ + "type": field_type, + "label": label, + "value": str(value), + }) + return fields + + +def append_custom_text_fields(fields: list, custom_fields: list) -> None: + """Append loaded custom fields using record-add's labelled custom syntax.""" + if not isinstance(custom_fields, list): + return + for field in custom_fields: + if not isinstance(field, dict): + continue + field_type = str(field.get("type") or "text") + label = str(field.get("label") or "").strip() + value = str(field.get("value") or "") + if not label or not value: + continue + safe_label = label.replace("=", "==") + if value.startswith("="): + value = f" {value}" + fields.append(f"c.{field_type}.{safe_label}={value}") + + class PamUserObject(): def __init__(self): self.folder_path = None # pam extend only @@ -997,6 +1040,7 @@ def __init__(self): self.connectDatabase = None self.managed = None self.oneTimeCode = None + self.custom = None self.attachments = None # fileRef self.scripts = None # script self.rotation_settings = None # DAG: rotation settings @@ -1024,6 +1068,7 @@ def load(cls, data: Union[str, dict], rotation_params: Optional[PamRotationParam obj.connectDatabase = str(data["connect_database"]) if "connect_database" in data else None obj.managed = utils.value_to_boolean(data["managed"]) if "managed" in data else None obj.oneTimeCode = str(data["otp"]) if "otp" in data else None + obj.custom = load_custom_text_fields(data.get("custom")) obj.attachments = PamAttachmentsObject.load(data.get("attachments", None)) obj.scripts = PamScriptsObject.load(data.get("scripts", None)) @@ -1062,6 +1107,7 @@ def create_record(self, params, folder_uid): if managed is not None: fields.append(f"f.checkbox.managed={str(managed).lower()}") if self.oneTimeCode: fields.append(f"f.oneTimeCode={self.oneTimeCode}") + append_custom_text_fields(fields, self.custom) files = self.attachments.attachments if self.attachments and isinstance(self.attachments, PamAttachmentsObject) else [] if files and isinstance(files, list): @@ -1070,7 +1116,8 @@ def create_record(self, params, folder_uid): fields.append(f"file=@{x.file}") if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid @@ -1165,7 +1212,8 @@ def create_record(self, params, folder_uid): fields.append(f"file=@{x.file}") if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid return uid @@ -1435,7 +1483,8 @@ def create_record(self, params, folder_uid): # switch to f.* once RT definition(s) update w/ pamSettings field if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid @@ -1628,7 +1677,8 @@ def create_record(self, params, folder_uid): # switch to f.* once RT definition(s) update w/ pamSettings field if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid @@ -1776,7 +1826,8 @@ def create_record(self, params, folder_uid): # switch to f.* once RT definition(s) update w/ pamSettings field if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid @@ -1881,7 +1932,8 @@ def create_record(self, params, folder_uid): # switch to f.* once RT definition(s) update w/ pamRemoteBrowserSettings field if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid diff --git a/keepercommander/commands/pam_import/cyberark_import.py b/keepercommander/commands/pam_import/cyberark_import.py index 163b4c691..950404a48 100644 --- a/keepercommander/commands/pam_import/cyberark_import.py +++ b/keepercommander/commands/pam_import/cyberark_import.py @@ -30,6 +30,8 @@ from prompt_toolkit import HTML, print_formatted_text from ..base import Command +from ..pam.vault_target import is_nested_share_folder, is_pam_nsf_record +from ... import api, vault, vault_extensions from ...display import bcolors from ...error import CommandError from ...importer.cyberark.cyberark_pam import ( @@ -62,6 +64,7 @@ RECORD_TYPE_PAM_MACHINE, RECORD_TYPE_PAM_DATABASE, RECORD_TYPE_PAM_DIRECTORY, + RECORD_TYPE_PAM_REMOTE_BROWSER, SCHEDULE_ON_DEMAND, ROTATION_UNMAPPED, ) @@ -70,12 +73,23 @@ IdempotencyDecision, PartitionSummary, RecordDecision, - annotate_record_with_marker, build_existing_index, partition_records, strip_id_marker, summarize, ) +from .nsf_helpers import find_pam_configuration, get_folder_record_uids +from .record_loader import iter_accessible_record_uids, load_pam_record + + +def is_matching_title(title: str, base: str) -> bool: + """True when *title* equals *base* or is ``{base} #N`` (case-insensitive).""" + title = title.casefold() + if title == base: + return True + if not title.startswith(base): + return False + return bool(re.fullmatch(r" #\d+", title[len(base):])) class SecureTempFileStore: @@ -160,6 +174,7 @@ class ImportRunOptions: user_map_file: str sync_mode: str = "upsert" strict_policies: bool = False + use_nsf: bool = False raw_kwargs: dict = field(default_factory=dict) @@ -597,31 +612,11 @@ def _map_single_account( "reason": f"unmappable platformId: {platform_id}", }) return - # Embed the CyberArk identity marker on every mapped record so - # future re-imports can match incoming accounts to already-created - # Keeper records (see importer/cyberark/pam/idempotency.py). The - # marker is a single line in ``notes`` — cheap to carry and - # survives the pam project import path unchanged (unlike ``custom`` - # fields, which PamBaseMachineParser does not preserve). - account_id = str(account.get("id", "") or "").strip() - if account_id: - annotate_record_with_marker(record, account_id, safe_name) - for nested in record.get("users") or []: - if not isinstance(nested, dict): - continue - # Nested pamUsers share the parent account's CyberArk id - # because CyberArk models the credential+resource as a - # single account. The marker is idempotent so re-runs - # can update the pamUser independently from its parent. - annotate_record_with_marker(nested, account_id, safe_name) self._apply_folder_paths(record, safe_name, folder_mapper) if password_failed: reason = "password retrieval failed" is_incomplete = True if is_incomplete: - note = f"INCOMPLETE: {reason}" - existing = (record.get("notes") or "").strip() - record["notes"] = f"{existing}\n{note}".strip() if existing else note incomplete.append(record) dual_fields = detect_dual_account(account) if dual_fields: @@ -641,6 +636,10 @@ def _map_single_account( if record.get("type") == RECORD_TYPE_LOGIN: pam_users.append(record) else: + if record.get("type") == RECORD_TYPE_PAM_REMOTE_BROWSER: + for user in record.pop("users", []) or []: + if isinstance(user, dict): + pam_users.append(user) if opts.skip_users: record.pop("users", None) pam_resources.append(record) @@ -696,6 +695,9 @@ def _collect_dependents(self, account: dict, record: dict, if not isinstance(dep, dict): continue dep["machine_title"] = machine_title + refs = dep.setdefault("machine_refs", []) + if isinstance(refs, list) and machine_title: + refs.append(machine_title) if dep.get("service_type") is None: unmapped_items.append({ "category": "CyberArk dependent", @@ -939,6 +941,7 @@ def _execute_vault_import(self, import_data: dict, self.params, import_data, opts.project_name, opts.config_uid, opts.batch_size, opts.batch_delay, mapped.pam_resources, mapped.pam_users, + use_nsf=opts.use_nsf, ) except Exception as e: logging.error("Import failed: %s", type(e).__name__) @@ -988,28 +991,22 @@ def _populate_folder_info(self, project_result: dict, project_name: str) -> None seen_uids: set = set() for wrapper_uid in wrapper_uids: - wrapper = self.params.folder_cache.get(wrapper_uid) - if not wrapper: - continue - for child_uid in getattr(wrapper, "subfolders", []) or []: - child = self.params.folder_cache.get(child_uid) - if not child or getattr(child, "type", "") != "shared_folder": + for child_uid, name in CyberArkPAMCleanupCommand._iter_project_child_folders( + self.params, wrapper_uid): + if child_uid in seen_uids: continue - if child.uid in seen_uids: - continue - seen_uids.add(child.uid) - name = getattr(child, "name", "") or "" + seen_uids.add(child_uid) if name == config_suffix: - config_folder_uid = child.uid + config_folder_uid = child_uid config_folder_name = name elif name == resources_suffix: - legacy_resources_uid = child.uid + legacy_resources_uid = child_uid legacy_resources_name = name elif name == users_suffix: - legacy_users_uid = child.uid + legacy_users_uid = child_uid legacy_users_name = name else: - safe_folders.append({"name": name, "uid": child.uid}) + safe_folders.append({"name": name, "uid": child_uid}) if config_folder_uid: folders_info["config_folder"] = config_folder_name @@ -1100,6 +1097,8 @@ def _maybe_auto_extend(self) -> None: return if opts.config_uid: return + if opts.use_nsf: + return if opts.dry_run or opts.output_file or opts.list_safes or opts.estimate_only: return # sync_down here so folder_cache / record_cache reflect any @@ -1139,6 +1138,8 @@ def _prepare_idempotency(self, mapped: MappedImportResult) -> Optional[dict]: opts = self.options if (opts.sync_mode or "").lower() == "create": return None + if opts.use_nsf and not opts.config_uid: + return None if opts.dry_run or opts.output_file: # Both paths skip the vault write; there's nothing to # de-dupe against and the pre-scan would only slow the @@ -1157,26 +1158,19 @@ def _prepare_idempotency(self, mapped: MappedImportResult) -> Optional[dict]: # pay the full-vault scan cost for nothing. return None - # Collect all shared folders directly under the project - # wrapper user folder(s). Records live inside these (and - # inside their Resources / Users subfolders for the - # safe-per-folder layout). + # Collect all shared / NSF folders directly under the project + # wrapper folder(s). Records live inside these (and inside + # their Resources / Users subfolders for the safe-per-folder + # layout). shared_folder_uids: list[str] = [] seen: set = set() for wrapper_uid in wrapper_uids: - wrapper = self.params.folder_cache.get(wrapper_uid) - if not wrapper: - continue - for child_uid in getattr(wrapper, "subfolders", []) or []: - child = self.params.folder_cache.get(child_uid) - if child is None: - continue - if getattr(child, "type", "") != "shared_folder": + for child_uid, _name in CyberArkPAMCleanupCommand._iter_project_child_folders( + self.params, wrapper_uid): + if child_uid in seen: continue - if child.uid in seen: - continue - seen.add(child.uid) - shared_folder_uids.append(child.uid) + seen.add(child_uid) + shared_folder_uids.append(child_uid) existing = build_existing_index(self.params, shared_folder_uids) if not existing.by_account_id and not existing.by_title: @@ -1619,9 +1613,7 @@ def _apply_service_dependent_mappings( }) continue - machine_record = self._find_machine_record( - dep.get("machine_address", ""), machine_index, - ) + machine_record = self._find_dependent_machine_record(dep, machine_index) if machine_record is None: summary["skipped_missing_machine"] += 1 summary["details"].append({ @@ -1704,6 +1696,29 @@ def _apply_service_dependent_mappings( return summary + @classmethod + def _find_dependent_machine_record(cls, dep: dict, machine_index: dict[str, Any]) -> Any: + refs = [] + if isinstance(dep, dict): + raw_refs = dep.get("machine_refs") + if isinstance(raw_refs, list): + refs.extend(raw_refs) + refs.extend([ + dep.get("machine_address", ""), + dep.get("machine_title", ""), + ]) + seen = set() + for ref in refs: + ref = str(ref or "").strip() + key = ref.casefold() + if not ref or key in seen: + continue + seen.add(key) + machine_record = cls._find_machine_record(ref, machine_index) + if machine_record is not None: + return machine_record + return None + def _build_record_indexes(self, mapped: MappedImportResult, vault, vault_extensions ) -> tuple[dict[str, Any], dict[str, Any]]: """Build ``(machine, user)`` record lookups from freshly imported vault data. @@ -1726,16 +1741,18 @@ def _build_record_indexes(self, mapped: MappedImportResult, vault, vault_extensi t = (u.get("title") or "").casefold() if t: imported_user_titles.add(t) - for rec in vault_extensions.find_records(self.params, record_version=3): + from .record_loader import iter_accessible_record_uids, load_pam_record + + for record_uid in iter_accessible_record_uids(self.params): + rec = load_pam_record(self.params, record_uid) + if rec is None: + continue rtype = getattr(rec, "record_type", "") or "" title = (getattr(rec, "title", "") or "").casefold() if rtype == RECORD_TYPE_PAM_MACHINE and title in imported_machine_titles: - loaded = vault.KeeperRecord.load(self.params, rec.record_uid) - if loaded is None: - continue - machine_index[title] = loaded + machine_index[title] = rec host_field = next( - (f for f in loaded.fields + (f for f in getattr(rec, "fields", []) or [] if (getattr(f, "type", "") or "") == "pamHostname" or (getattr(f, "label", "") or "").lower() == "host"), None, @@ -1748,11 +1765,9 @@ def _build_record_indexes(self, mapped: MappedImportResult, vault, vault_extensi elif isinstance(raw, str): host_str = raw if host_str: - machine_index[host_str.casefold()] = loaded + machine_index[host_str.casefold()] = rec elif rtype == "pamUser" and title in imported_user_titles: - loaded = vault.KeeperRecord.load(self.params, rec.record_uid) - if loaded is not None: - user_index[title] = loaded + user_index[title] = rec return machine_index, user_index @staticmethod @@ -1922,6 +1937,8 @@ class CyberArkPAMImportCommand(Command): # Self-hosted with mutual TLS client cert (P12) pam project cyberark-import pvwa.internal.com --client-cert-p12 ./client.p12 --name "Internal" + # Import into Nested Share Folders (NSF) + pam project cyberark-import pvwa.example.com --name "PAM Migration" --nsf ''') parser.add_argument("server", action="store", help="CyberArk PVWA host (e.g. mycompany.cyberark.cloud or pvwa.example.com)") parser.add_argument("--name", "-n", required=False, dest="project_name", action="store", @@ -1938,7 +1955,12 @@ class CyberArkPAMImportCommand(Command): "control. 'ksm'/'exact' nest safe-named subfolders under the " "legacy Resources/Users shared folders. 'flat' puts every " "record into the two legacy shared folders with aggregated " - "permissions.") + "permissions. With --nsf, the same layout is created using " + "Nested Share Folders instead of classic shared folders.") + parser.add_argument("--nsf", required=False, dest="use_nsf", action="store_true", + default=False, + help="Create project folders and records in Nested Share Folders " + "(folders, records, rotation, PAM config, and permissions).") parser.add_argument("--safes", required=False, dest="safes", action="store", default="", help="Include only matching Safes (comma-separated, supports globs)") parser.add_argument("--exclude-safes", required=False, dest="exclude_safes", action="store", @@ -2126,6 +2148,7 @@ def execute(self, params, **kwargs): user_map_file=kwargs.get("user_map", ""), sync_mode=(kwargs.get("sync_mode") or "upsert").lower(), strict_policies=bool(kwargs.get("strict_policies", False)), + use_nsf=kwargs.get("use_nsf", False) is True, raw_kwargs=kwargs, ) CyberArkImportOrchestrator(self, params, client, options).run() @@ -2134,7 +2157,8 @@ def execute(self, params, **kwargs): def _execute_import(self, params, import_data: dict, project_name: str, config_uid: str, batch_size: int, batch_delay: float, - resources: list[dict], users: list[dict]) -> Optional[dict]: + resources: list[dict], users: list[dict], + use_nsf: bool = False) -> Optional[dict]: """Execute the vault import using pam project import/extend commands.""" from .edit import PAMProjectImportCommand from .extend import PAMProjectExtendCommand @@ -2145,17 +2169,18 @@ def _execute_import(self, params, import_data: dict, project_name: str, if total_records <= batch_size: # Single batch — use import or extend directly return self._single_batch_import( - params, import_data, project_name, config_uid + params, import_data, project_name, config_uid, use_nsf=use_nsf, ) else: # Multi-batch: first batch creates project, remaining extend return self._multi_batch_import( params, import_data, project_name, config_uid, - resources, users, batch_size, batch_delay, + resources, users, batch_size, batch_delay, use_nsf=use_nsf, ) def _single_batch_import(self, params, import_data: dict, - project_name: str, config_uid: str) -> dict: + project_name: str, config_uid: str, + use_nsf: bool = False) -> dict: """Import all records in a single batch.""" from .edit import PAMProjectImportCommand from .extend import PAMProjectExtendCommand @@ -2163,12 +2188,18 @@ def _single_batch_import(self, params, import_data: dict, tmp_path = _temp_store.write_json(import_data) try: if config_uid: + # Extend auto-detects NSF from the existing project tree. + if use_nsf: + logging.info( + "--nsf is ignored when extending an existing project " + "(--config); Nested Share Folders are detected automatically.") PAMProjectExtendCommand().execute( - params, config=config_uid, file_name=tmp_path, dry_run=False + params, config=config_uid, file_name=tmp_path, dry_run=False, ) else: PAMProjectImportCommand().execute( - params, project_name=project_name, file_name=tmp_path, dry_run=False + params, project_name=project_name, file_name=tmp_path, + dry_run=False, use_nsf=use_nsf, ) finally: _temp_store.remove(tmp_path) @@ -2179,7 +2210,8 @@ def _single_batch_import(self, params, import_data: dict, def _multi_batch_import(self, params, import_data: dict, project_name: str, config_uid: str, resources: list[dict], users: list[dict], - batch_size: int, batch_delay: float) -> dict: + batch_size: int, batch_delay: float, + use_nsf: bool = False) -> dict: """Import records in multiple batches with adaptive throttling.""" from .edit import PAMProjectImportCommand from .extend import PAMProjectExtendCommand @@ -2207,7 +2239,8 @@ def _multi_batch_import(self, params, import_data: dict, tmp_path = _temp_store.write_json(first_batch_data) try: PAMProjectImportCommand().execute( - params, project_name=project_name, file_name=tmp_path, dry_run=False + params, project_name=project_name, file_name=tmp_path, + dry_run=False, use_nsf=use_nsf, ) finally: _temp_store.remove(tmp_path) @@ -2217,13 +2250,13 @@ def _multi_batch_import(self, params, import_data: dict, if not config_uid: config_uid = self._find_config_uid(params, project_name) else: - # Subsequent batches: extend + # Subsequent batches: extend (NSF auto-detected from project) extend_data = build_extend_json(batch_resources, batch_users) tmp_path = _temp_store.write_json(extend_data) try: if config_uid: PAMProjectExtendCommand().execute( - params, config=config_uid, file_name=tmp_path, dry_run=False + params, config=config_uid, file_name=tmp_path, dry_run=False, ) else: logging.error("Cannot extend: PAM configuration UID not found after initial import") @@ -2240,17 +2273,33 @@ def _multi_batch_import(self, params, import_data: dict, def _find_config_uid(self, params, project_name: str) -> str: """Find PAM configuration UID by project name after initial import. - Handles #N suffix deduplication from PAMProjectImportCommand.""" - from ... import api, vault_extensions - + Handles #N suffix deduplication from PAMProjectImportCommand. + Searches classic vault records and Nested Share Folder caches. + """ api.sync_down(params) config_base = f"{project_name} Configuration".casefold() candidates = [] + + # Classic path (keeps existing unit-test mocks working) for c in vault_extensions.find_records(params, record_version=6): - t = c.title.casefold() - if t == config_base or (t.startswith(config_base) and re.match(r' #\d+$', t[len(config_base):])): + if is_matching_title(getattr(c, "title", "") or "", config_base): candidates.append(c) + + # NSF / full-access path — pick up configs that live only in NSF caches + seen_uids = {getattr(c, "record_uid", "") for c in candidates} + for uid in iter_accessible_record_uids(params): + if uid in seen_uids: + continue + rec = load_pam_record(params, uid) + if rec and getattr(rec, "version", None) == 6: + if is_matching_title(getattr(rec, "title", "") or "", config_base): + candidates.append(rec) + seen_uids.add(uid) + if not candidates: + exact = find_pam_configuration(params, f"{project_name} Configuration") + if exact: + return exact.record_uid logging.warning(f"PAM configuration not found for project '{project_name}' after import") return "" # Prefer highest suffix number (most recently created) @@ -2364,7 +2413,7 @@ def _interactive_safe_picker(safes: list[dict]) -> Optional[list[str]]: class CyberArkPAMCleanupCommand(Command): - """Remove a CyberArk-imported project: records, folders, gateway, KSM app.""" + """Remove a CyberArk-imported project: records, folders, and PAM config.""" parser = argparse.ArgumentParser( prog="pam project cyberark-cleanup", @@ -2402,16 +2451,13 @@ def execute(self, params, **kwargs): raise CommandError("pam project cyberark-cleanup", "Either --name or --config is required") - from ... import api, utils, vault, vault_extensions - from ..pam import gateway_helper - from ..pam.config_helper import configuration_controller_get - from ...loginv3 import CommonHelperMethods - api.sync_down(params) - # Find PAM config by name or UID + # Find PAM config by name or UID (classic + NSF) if config_uid: config_rec = vault.KeeperRecord.load(params, config_uid) + if not config_rec: + config_rec = load_pam_record(params, config_uid) if not config_rec: raise CommandError("pam project cyberark-cleanup", f"PAM config record '{config_uid}' not found") @@ -2420,51 +2466,39 @@ def execute(self, params, **kwargs): config_base = f"{project_name} Configuration".casefold() config_rec = None for c in vault_extensions.find_records(params, record_version=6): - if c.title.casefold().startswith(config_base): + if is_matching_title(getattr(c, "title", "") or "", config_base): config_rec = c config_uid = c.record_uid break + if not config_rec: + for uid in iter_accessible_record_uids(params): + rec = load_pam_record(params, uid) + if rec and getattr(rec, "version", None) == 6: + if is_matching_title(getattr(rec, "title", "") or "", config_base): + config_rec = rec + config_uid = uid + break + if not config_rec: + exact = find_pam_configuration(params, f"{project_name} Configuration") + if exact: + config_rec = exact + config_uid = exact.record_uid if not config_rec: raise CommandError("pam project cyberark-cleanup", f"PAM config for project '{project_name}' not found") - # Resolve gateway linked to this PAM config - gateway_uid = None - gateway_name = None - gw_match = None - try: - controller = configuration_controller_get( - params, CommonHelperMethods.url_safe_str_to_bytes( - config_rec.record_uid)) - if controller and controller.controllerUid: - gateway_uid = controller.controllerUid - all_gw = gateway_helper.get_all_gateways(params) - gw_match = next((g for g in all_gw - if g.controllerUid == gateway_uid), None) - if gw_match: - gateway_name = gw_match.controllerName - except Exception as e: - logging.debug("Could not resolve gateway: %s", e) - - # Resolve KSM application linked to the gateway - ksm_app_uid = None - ksm_app_name = None - if gw_match and gw_match.applicationUid: - ksm_app_uid = utils.base64_url_encode(gw_match.applicationUid) - app_rec = vault.KeeperRecord.load(params, ksm_app_uid) - if app_rec: - ksm_app_name = getattr(app_rec, "title", ksm_app_uid) - - # Find shared folders. The new safe-per-folder layout creates one - # shared folder per CyberArk safe under the project wrapper folder + # Find shared / NSF folders. The new safe-per-folder layout creates one + # folder per CyberArk safe under the project wrapper folder # plus an admin Config folder; each safe folder has two - # ``Resources``/``Users`` shared_folder_folder subfolders that + # ``Resources``/``Users`` subfolders that # hold the records. The legacy layout creates exactly two folders # ("{project} - Resources" and "{project} - Users") with safe-named # subfolders inside. Discover everything by walking the project - # wrapper user-folder under PAM Environments so cleanup handles - # both shapes (and any subset thereof) without hardcoding names. + # wrapper under PAM Environments so cleanup handles + # both classic shared folders and Nested Share Folders + # (and any subset thereof) without hardcoding names. sf_uids = [] + record_count = 0 sf_names: list = [] all_record_uids: set = set() res_name = f"{project_name} - Resources" @@ -2474,68 +2508,40 @@ def execute(self, params, **kwargs): def _collect_records_recursive(folder_uid: str): """Walk the folder subtree and accumulate every record UID (records living directly in this folder + records living in - any descendant ``shared_folder_folder``).""" + any descendant classic or NSF subfolder).""" stack = [folder_uid] + visited: set = set() while stack: fuid = stack.pop() - for ruid in params.subfolder_record_cache.get(fuid, set()) or set(): - all_record_uids.add(ruid) + if not fuid or fuid in visited: + continue + visited.add(fuid) + all_record_uids.update(get_folder_record_uids(params, fuid)) folder = params.folder_cache.get(fuid) if not folder: + # NSF-only parents may still appear in nested_share_folders + nsf = getattr(params, "nested_share_folders", None) or {} + for child_uid, info in nsf.items(): + if (info.get("parent_uid") or None) == fuid: + stack.append(child_uid) continue for sub_uid in getattr(folder, "subfolders", []) or []: stack.append(sub_uid) - def _delete_folder(folder_uid: str) -> bool: - folder = params.folder_cache.get(folder_uid) - if not folder: - return False - del_obj = { - "delete_resolution": "unlink", - "object_uid": folder.uid, - "object_type": folder.type, - } - parent = params.folder_cache.get(folder.parent_uid) - if parent: - del_obj["from_uid"] = parent.uid - del_obj["from_type"] = parent.type - else: - del_obj["from_type"] = "user_folder" - rq = {"command": "pre_delete", "objects": [del_obj]} - rs = api.communicate(params, rq) - if rs.get("result") != "success": - return False - pdr = rs.get("pre_delete_response", {}) - del_rq = { - "command": "delete", - "pre_delete_token": pdr.get("pre_delete_token", ""), - } - api.communicate(params, del_rq) - return True - project_folder_uids = self._find_project_wrapper_folder_uids( params, project_name, ) if project_folder_uids: sf_uids_seen: set = set() for project_uid in project_folder_uids: - project_folder = params.folder_cache.get(project_uid) - if not project_folder: - continue - for child_uid in getattr(project_folder, "subfolders", []) or []: - child = params.folder_cache.get(child_uid) - if not child: - continue - # Only collect shared folders that we created — i.e. - # the type is SharedFolderType. - if getattr(child, "type", "") != "shared_folder": + for child_uid, child_name in self._iter_project_child_folders( + params, project_uid): + if child_uid in sf_uids_seen: continue - if child.uid in sf_uids_seen: - continue - sf_uids_seen.add(child.uid) - sf_uids.append(child.uid) - sf_names.append(getattr(child, "name", "") or "") - _collect_records_recursive(child.uid) + sf_uids_seen.add(child_uid) + sf_uids.append(child_uid) + sf_names.append(child_name) + _collect_records_recursive(child_uid) else: # Fallback: scan the shared-folder cache by name. Catches the # legacy two-folder layout when the project wrapper folder was @@ -2546,25 +2552,24 @@ def _delete_folder(folder_uid: str) -> bool: sf_uids.append(sf_uid) sf_names.append(name) _collect_records_recursive(sf_uid) - - # Ensure PAM config is included even if it lives outside the - # discovered shared-folder tree. - if config_uid: - all_record_uids.add(config_uid) + # NSF fallback by name under nested_share_folders + for nsf_uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + name = info.get("name", "") or "" + if name in (res_name, usr_name, config_name) and nsf_uid not in sf_uids: + sf_uids.append(nsf_uid) + sf_names.append(name) + _collect_records_recursive(nsf_uid) + record_count = len(all_record_uids) print(f"\nCyberArk PAM Project Cleanup") print("=" * 50) - print(f" Project: {project_name}") - print(f" PAM Config: {config_uid}") - print(f" Gateway: {gateway_name or '(not found)'}") - print(f" KSM App: {ksm_app_name or ksm_app_uid or '(not found)'}") - print(f" Folders: {len(sf_uids)}") + print(f" Project: {project_name}") + print(f" Config: {config_uid}") + print(f" Folders: {len(sf_uids)}") for sf_name in sf_names: if sf_name: print(f" • {sf_name}") - if project_folder_uids: - print(f" Wrappers: {len(project_folder_uids)}") - print(f" Records: {len(all_record_uids)}") + print(f" Records: ~{record_count}") if dry_run: print(" (dry run — no changes made)") @@ -2572,131 +2577,205 @@ def _delete_folder(folder_uid: str) -> bool: return if not auto_confirm: - answer = input("\n Delete all of the above? [y/N]: ").strip().lower() + answer = input("\n Delete project folders (and all records inside)? [y/N]: ").strip().lower() if answer not in ("y", "yes"): print(" Cancelled.") return - deleted = 0 - failed = 0 - - # Delete records in batches (same API as api.delete_record) - if all_record_uids: - logging.warning("Deleting %d records...", len(all_record_uids)) - uid_list = list(all_record_uids) - batch_size = 50 - for i in range(0, len(uid_list), batch_size): - batch = uid_list[i:i + batch_size] - try: - rq = {"command": "record_update", "delete_records": batch} - api.communicate(params, rq) - deleted += len(batch) - except Exception as e: - failed += len(batch) - logging.warning("Failed to delete record batch: %s", e) - - # Delete shared folders (safe folders + Config folder) - if sf_uids: - logging.warning("Removing %d shared folder(s)...", len(sf_uids)) - for sf_uid in sf_uids: - try: - if not _delete_folder(sf_uid): - failed += 1 - logging.warning("Failed to remove shared folder %s", - sf_uid) - except Exception as e: - failed += 1 - logging.warning("Failed to remove shared folder %s: %s", - sf_uid, e) - - # Delete project wrapper user-folder(s) under PAM Environments - if project_folder_uids: - logging.warning("Removing %d project wrapper folder(s)...", - len(project_folder_uids)) - for wrapper_uid in project_folder_uids: - try: - if not _delete_folder(wrapper_uid): - failed += 1 - logging.warning("Failed to remove wrapper folder %s", - wrapper_uid) - except Exception as e: - failed += 1 - logging.warning("Failed to remove wrapper folder %s: %s", - wrapper_uid, e) - - # Remove gateway - if gateway_uid: - logging.warning("Removing gateway \"%s\"...", - gateway_name or gateway_uid) + from ..folder import FolderRemoveCommand + from ..record import RecordRemoveCommand + from ..nested_share_folder.folder_commands import NestedShareFolderRemoveCommand + from ..nested_share_folder.record_commands import NestedShareRecordRemoveCommand + + # Prefer folder deletion: records (and nested subfolders) are removed + # with the folder tree. Delete content folders first, then wrappers. + folders_deleted = 0 + folders_failed = 0 + folders_to_remove = list(sf_uids) + list(project_folder_uids) + for folder_uid in folders_to_remove: try: - gateway_helper.remove_gateway(params, gateway_uid) + if is_nested_share_folder(params, folder_uid): + NestedShareFolderRemoveCommand().execute( + params, folders=[folder_uid], force=True, + operation="folder-trash", quiet=True) + else: + FolderRemoveCommand().execute( + params, pattern=[folder_uid], force=True, quiet=True) + folders_deleted += 1 except Exception as e: - failed += 1 - logging.warning("Failed to remove gateway: %s", e) + folders_failed += 1 + logging.warning("Failed to delete folder %s: %s", + folder_uid, type(e).__name__) + + api.sync_down(params) - # Remove KSM application - if ksm_app_uid: - logging.warning("Removing KSM app \"%s\"...", - ksm_app_name or ksm_app_uid) + # PAM config may already be gone with its Config folder; remove only + # if it survived (e.g. legacy layout / config outside project folders). + config_deleted = False + if config_uid and ( + config_uid in (params.record_cache or {}) + or config_uid in (getattr(params, "nested_share_records", None) or {}) + or vault.KeeperRecord.load(params, config_uid) + or load_pam_record(params, config_uid)): try: - from ..ksm import KSMCommand - KSMCommand.remove_v5_app(params, ksm_app_uid, - purge=True, force=True) + if is_pam_nsf_record(params, config_uid): + NestedShareRecordRemoveCommand().execute( + params, records=[config_uid], force=True, + operation="owner-trash") + else: + RecordRemoveCommand().execute( + params, force=True, record=config_uid) + config_deleted = True + api.sync_down(params) except Exception as e: - failed += 1 - logging.warning("Failed to remove KSM app: %s", e) - - api.sync_down(params) - msg = f"\nCleanup complete: {deleted} records deleted" - if failed: - msg += f" ({failed} failed — see warnings above)" + folders_failed += 1 + logging.warning("Failed to delete PAM config record %s: %s", + config_uid, type(e).__name__) + + msg = f"\nCleanup complete: {folders_deleted} folders deleted" + if config_deleted: + msg += " (PAM config removed)" + if folders_failed: + msg += f" ({folders_failed} failed — see warnings above)" print(msg) print("=" * 50) PAM_ROOT_FOLDER_NAME = "PAM Environments" + @classmethod + def _is_project_content_folder(cls, params, folder_uid: str, folder=None) -> bool: + """True for classic shared folders or Nested Share Folders under a project.""" + from ...subfolder import BaseFolderNode + + if is_nested_share_folder(params, folder_uid): + return True + folder = folder or (params.folder_cache or {}).get(folder_uid) + if not folder: + return False + ftype = getattr(folder, "type", "") or "" + return ftype in ( + BaseFolderNode.SharedFolderType, + BaseFolderNode.NestedShareFolderType, + ) + + @classmethod + def _iter_project_child_folders(cls, params, wrapper_uid: str): + """Yield ``(uid, name)`` for content folders under a project wrapper. + + Includes classic shared folders and Nested Share Folders so + report / idempotency / cleanup share one discovery path. + """ + seen: set = set() + folder_cache = getattr(params, "folder_cache", None) or {} + wrapper = folder_cache.get(wrapper_uid) + if wrapper: + for child_uid in getattr(wrapper, "subfolders", []) or []: + if child_uid in seen: + continue + child = folder_cache.get(child_uid) + if not child or not cls._is_project_content_folder(params, child_uid, child): + continue + seen.add(child_uid) + yield child_uid, getattr(child, "name", "") or "" + + # NSF children may exist in nested_share_folders before folder_cache + # parent.subfolders is fully linked after a partial sync. + for child_uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + if child_uid in seen: + continue + if (info.get("parent_uid") or None) != wrapper_uid: + continue + if not cls._is_project_content_folder(params, child_uid): + continue + seen.add(child_uid) + yield child_uid, info.get("name", "") or "" + @classmethod def _find_project_wrapper_folder_uids(cls, params, project_name: str) -> list: - """Return UIDs of every project wrapper user-folder under + """Return UIDs of every project wrapper folder under ``PAM Environments`` whose name matches ``project_name`` (or ``project_name #N`` when the project was imported multiple times). - The wrapper folder is a *user folder* (not shared), and per-safe - shared folders are created as direct children of it. Returning a - list keeps cleanup correct in the rare case where two projects - share a name (PAMProjectImportCommand allows duplicates via the - ``#N`` suffix). + Supports classic user-folder wrappers and Nested Share Folder + wrappers created with ``--nsf``. Returning a list keeps cleanup + correct in the rare case where two projects share a name + (PAMProjectImportCommand allows duplicates via the ``#N`` suffix). """ + from ...subfolder import BaseFolderNode + wrapper_uids: list = [] folders = params.folder_cache if params and params.folder_cache else {} if not isinstance(folders, dict): - return wrapper_uids + folders = {} - # Locate root "PAM Environments" user folder(s). + # Locate root "PAM Environments" folder(s) — classic user folder + # and/or Nested Share Folder. root_uids: list = [] + seen_roots: set = set() for uid, f in folders.items(): if not f or getattr(f, "parent_uid", None): continue - if getattr(f, "type", "") != "user_folder": + ftype = getattr(f, "type", "") or "" + if ftype not in ( + BaseFolderNode.UserFolderType, + BaseFolderNode.NestedShareFolderType, + ): + continue + if getattr(f, "name", "") != cls.PAM_ROOT_FOLDER_NAME: + continue + root_uids.append(uid) + seen_roots.add(uid) + + for uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + if uid in seen_roots: + continue + if (info.get("parent_uid") or None) is not None: + continue + if info.get("name", "") != cls.PAM_ROOT_FOLDER_NAME: continue - if getattr(f, "name", "") == cls.PAM_ROOT_FOLDER_NAME: - root_uids.append(uid) + root_uids.append(uid) + seen_roots.add(uid) + if not root_uids: return wrapper_uids # PAMProjectImportCommand emits "{project_name}" or - # "{project_name} #N" for the wrapper user folder, so match both - # shapes here. + # "{project_name} #N" for the wrapper folder, so match both + # shapes here (classic user folder or NSF). base = project_name + name_re = re.compile(rf"^{re.escape(base)} #\d+$") + seen_wrappers: set = set() + + def _maybe_add_wrapper(uid: str, name: str, folder=None) -> None: + if uid in seen_wrappers: + return + if name != base and not name_re.match(name): + return + ftype = getattr(folder, "type", "") if folder else "" + if folder is not None: + if ftype not in ( + BaseFolderNode.UserFolderType, + BaseFolderNode.NestedShareFolderType, + ): + return + elif not is_nested_share_folder(params, uid): + return + wrapper_uids.append(uid) + seen_wrappers.add(uid) + for root_uid in root_uids: root_folder = folders.get(root_uid) - if not root_folder: - continue - for child_uid in getattr(root_folder, "subfolders", []) or []: - child = folders.get(child_uid) - if not child or getattr(child, "type", "") != "user_folder": + if root_folder: + for child_uid in getattr(root_folder, "subfolders", []) or []: + child = folders.get(child_uid) + if not child: + continue + name = getattr(child, "name", "") or "" + _maybe_add_wrapper(child_uid, name, child) + + for child_uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + if (info.get("parent_uid") or None) != root_uid: continue - name = getattr(child, "name", "") or "" - if name == base or re.match(rf"^{re.escape(base)} #\d+$", name): - wrapper_uids.append(child.uid) + _maybe_add_wrapper(child_uid, info.get("name", "") or "") + return wrapper_uids diff --git a/keepercommander/commands/pam_import/edit.py b/keepercommander/commands/pam_import/edit.py index e2c6420ef..2cd136053 100644 --- a/keepercommander/commands/pam_import/edit.py +++ b/keepercommander/commands/pam_import/edit.py @@ -445,10 +445,22 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, # record. Cannot live in any safe folder, or the safe's members # would gain access to the central config record. config_folder_name = f"""{res["project_folder"]} - Config""" + + # Verify principals once (avoids one round-trip per safe). + if all_user_perms: + self.verify_users_and_teams(params, all_user_perms) + + if use_nsf: + self._create_safe_folders_nsf_batch( + params, project_folder_uid, res, safe_folder_records, + config_folder_name, safe_folder_map, + ) + return + config_uid = self.create_subfolder( params, folder_name=config_folder_name, parent_uid=project_folder_uid, permissions=dict(default_fperm), - use_nsf=use_nsf, + use_nsf=False, ) res["resources_folder"] = config_folder_name res["users_folder"] = config_folder_name @@ -457,10 +469,6 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, res["config_folder_uid"] = config_uid res["config_folder"] = config_folder_name - # Verify principals once (avoids one round-trip per safe). - if all_user_perms: - self.verify_users_and_teams(params, all_user_perms) - # Create one shared folder per safe under the project wrapper and # apply its specific permission set. Inside each safe folder, # create two organizational subfolders named @@ -477,7 +485,7 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, folder_uid = self.create_subfolder( params, folder_name=record["name"], parent_uid=project_folder_uid, permissions=record["fperm"], - use_nsf=use_nsf, + use_nsf=False, ) # Top-level lookup key (no slash) maps to the safe folder # itself for callers that still emit ``folder_path = ""`` @@ -488,11 +496,11 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, usr_sub_name = f"{record['name']} - Users" res_sub_uid = self.create_subfolder( params, folder_name=res_sub_name, parent_uid=folder_uid, - use_nsf=use_nsf, + use_nsf=False, ) usr_sub_uid = self.create_subfolder( params, folder_name=usr_sub_name, parent_uid=folder_uid, - use_nsf=use_nsf, + use_nsf=False, ) safe_folder_map[f"{record['name']}/{res_sub_name}"] = res_sub_uid safe_folder_map[f"{record['name']}/{usr_sub_name}"] = usr_sub_uid @@ -511,6 +519,101 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, res["safe_folder_map"] = safe_folder_map + def _create_safe_folders_nsf_batch(self, params, project_folder_uid: str, + res: dict, safe_folder_records: list, + config_folder_name: str, + safe_folder_map: dict) -> None: + """Create Config + per-safe NSF folders via batched folder_add_v3. + + Two layers (parents must exist before children): + 1. Config folder + each safe folder under the project + 2. ``{safe} - Resources`` / ``{safe} - Users`` under each safe + """ + from .nsf_helpers import create_nsf_folders_batch + + # Layer 1: config + safe wrappers (same parent) + layer1_specs = [ + {"name": config_folder_name, "parent_uid": project_folder_uid}, + ] + for record in safe_folder_records: + layer1_specs.append({ + "name": record["name"], + "parent_uid": project_folder_uid, + }) + + layer1 = create_nsf_folders_batch( + params, layer1_specs, sync=False, command='pam-project-import', + ) + config_uid = layer1[0]["folder_uid"] + res["resources_folder"] = config_folder_name + res["users_folder"] = config_folder_name + res["resources_folder_uid"] = config_uid + res["users_folder_uid"] = config_uid + res["config_folder_uid"] = config_uid + res["config_folder"] = config_folder_name + + safe_uids: list = [] + for i, record in enumerate(safe_folder_records): + folder_uid = layer1[i + 1]["folder_uid"] + safe_uids.append(folder_uid) + safe_folder_map[record["name"]] = folder_uid + + # Layer 2: Resources / Users children under each safe + layer2_specs: list = [] + layer2_meta: list = [] # (safe_index, kind, sub_name) + for i, record in enumerate(safe_folder_records): + res_sub_name = f"{record['name']} - Resources" + usr_sub_name = f"{record['name']} - Users" + parent_uid = safe_uids[i] + layer2_specs.append({"name": res_sub_name, "parent_uid": parent_uid}) + layer2_meta.append((i, "resources", res_sub_name)) + layer2_specs.append({"name": usr_sub_name, "parent_uid": parent_uid}) + layer2_meta.append((i, "users", usr_sub_name)) + + layer2 = [] + if layer2_specs: + layer2 = create_nsf_folders_batch( + params, layer2_specs, sync=True, command='pam-project-import', + ) + else: + # Config-only project: still need one sync after layer 1. + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) + + # Assemble maps / safe_folders entries + by_safe: dict = { + i: {"resources_uid": None, "users_uid": None, + "resources_name": None, "users_name": None} + for i in range(len(safe_folder_records)) + } + for result, (safe_i, kind, sub_name) in zip(layer2, layer2_meta): + uid = result["folder_uid"] + parent_name = safe_folder_records[safe_i]["name"] + safe_folder_map[f"{parent_name}/{sub_name}"] = uid + if kind == "resources": + by_safe[safe_i]["resources_uid"] = uid + by_safe[safe_i]["resources_name"] = sub_name + else: + by_safe[safe_i]["users_uid"] = uid + by_safe[safe_i]["users_name"] = sub_name + + for i, record in enumerate(safe_folder_records): + folder_uid = safe_uids[i] + info = by_safe[i] + res["safe_folders"].append({ + "name": record["name"], + "safe_name": record["safe_name"], + "uid": folder_uid, + "resources_subfolder": info["resources_name"], + "resources_subfolder_uid": info["resources_uid"], + "users_subfolder": info["users_name"], + "users_subfolder_uid": info["users_uid"], + }) + if record["uperm"]: + self.add_folder_permissions(params, folder_uid, record["uperm"]) + + res["safe_folder_map"] = safe_folder_map + def process_ksm_app(self, params, project: dict) -> dict: res = { "app_name_target": "", @@ -1039,17 +1142,14 @@ def create_subfolder(self, params, folder_name:str, parent_uid:str="", permissio name = str(folder_name or "").strip() if use_nsf or is_nested_share_folder(params, parent_uid): - from .nsf_helpers import seed_nsf_folder_cache, sync_down_preserving_nsf_keys - from ...nested_share_folder.folder_api import create_folder_v3 - result = create_folder_v3(params, name, parent_uid=parent_uid or None) - if isinstance(result, dict) and result.get('success') is False: - raise CommandError("pam", result.get('message') or 'Failed to create Nested Share Folder') - folder_uid = result.get('folder_uid') if isinstance(result, dict) else None - if not folder_uid: - raise CommandError("pam", f'Nested Share Folder creation did not return UID: {name}') - folder_key = result.get('folder_key_unencrypted') if isinstance(result, dict) else None - seed_nsf_folder_cache(params, folder_uid, name, parent_uid or None, folder_key) - sync_down_preserving_nsf_keys(params) + from .nsf_helpers import create_nsf_folders_batch + results = create_nsf_folders_batch( + params, + [{"name": name, "parent_uid": parent_uid or None}], + sync=True, + command='pam', + ) + folder_uid = results[0]["folder_uid"] params.environment_variables[LAST_FOLDER_UID] = folder_uid return folder_uid @@ -1536,6 +1636,9 @@ def _resolve_folder_uid(obj, default_uid: str) -> str: logging.warning(f"Processing external users: {len(users)}") for n, user in enumerate(users): # standalone users user.create_record(params, _resolve_folder_uid(user, shfusr)) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if n % pdelta == 0: print(f"{n}/{len(users)}") print(f"{len(users)}/{len(users)}\n") @@ -1549,6 +1652,11 @@ def _resolve_folder_uid(obj, default_uid: str) -> str: admin_uid = get_admin_credential(mach, True) mach_folder_uid = _resolve_folder_uid(mach, shfres) mach.create_record(params, mach_folder_uid) + # NSF creates with sync_after=False; pam tunnel edit resolves from + # record_cache, so sync once when the new UID is not loaded yet. + if mach.uid and mach.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) tdag.link_resource_to_config(mach.uid) if isinstance(mach, PamRemoteBrowserObject): # RBI args = parse_command_options(mach, True) @@ -1621,6 +1729,9 @@ def _resolve_folder_uid(obj, default_uid: str) -> str: # to every record originating from that CyberArk safe. user_folder_uid = _resolve_folder_uid(user, mach_folder_uid) user.create_record(params, user_folder_uid) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if isinstance(user, PamUserObject): # rotation setup tdag.link_user_to_resource(user.uid, mach.uid, admin_uid==user.uid, True) if user.rotation_settings: @@ -1712,4 +1823,10 @@ def _resolve_folder_uid(obj, default_uid: str) -> str: else: logging.debug(f"Unable to resolve domain admin '{pce.dom_administrative_credential}' for PAM Domain configuration.") + # One sync after bulk NSF record creates (create_record uses sync_after=False). + use_nsf = project["options"].get("use_nsf", False) is True + if use_nsf or is_nested_share_folder(params, shfres) or is_nested_share_folder(params, shfusr): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) + logging.debug("Done processing project data.") diff --git a/keepercommander/commands/pam_import/extend.py b/keepercommander/commands/pam_import/extend.py index ecc846e04..f2b1a208b 100644 --- a/keepercommander/commands/pam_import/extend.py +++ b/keepercommander/commands/pam_import/extend.py @@ -1426,6 +1426,9 @@ def process_data(self, params, project): for n, user in enumerate(new_users): folder_uid = getattr(user, "resolved_folder_uid", None) or shfusr extend_create_record(params, user, folder_uid) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if n % pdelta == 0: print(f"{n}/{len(new_users)}") print(f"{len(new_users)}/{len(new_users)}\n") @@ -1441,6 +1444,11 @@ def process_data(self, params, project): folder_uid = getattr(mach, "resolved_folder_uid", None) or shfres admin_uid = get_admin_credential(mach, True) extend_create_record(params, mach, folder_uid) + # NSF creates with sync_after=False; pam tunnel edit resolves from + # record_cache / NSF caches, so sync when the new UID is not loaded yet. + if mach.uid and mach.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) tdag.link_resource_to_config(mach.uid) if isinstance(mach, PamRemoteBrowserObject): args = parse_command_options(mach, True) @@ -1508,6 +1516,9 @@ def process_data(self, params, project): rs.resourceUid = mach.uid ufolder = getattr(user, "resolved_folder_uid", None) or shfusr extend_create_record(params, user, ufolder) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if isinstance(user, PamUserObject): tdag.link_user_to_resource(user.uid, mach.uid, admin_uid == user.uid, True) if rs: @@ -1546,6 +1557,9 @@ def process_data(self, params, project): rs.resourceUid = mach.uid ufolder = getattr(user, "resolved_folder_uid", None) or shfusr extend_create_record(params, user, ufolder) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if isinstance(user, PamUserObject): tdag.link_user_to_resource(user.uid, mach.uid, admin_uid == user.uid, True) if rs: @@ -1628,5 +1642,11 @@ def process_data(self, params, project): if refs: api.sync_down(params) add_pam_scripts(params, pam_cfg_uid, refs) + + # One sync after bulk NSF record creates (create_record uses sync_after=False). + if is_nested_share_folder(params, shfres) or is_nested_share_folder(params, shfusr): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) + logging.debug("Done processing project data.") return diff --git a/keepercommander/commands/pam_import/nsf_helpers.py b/keepercommander/commands/pam_import/nsf_helpers.py index 4088daa93..6cc823857 100644 --- a/keepercommander/commands/pam_import/nsf_helpers.py +++ b/keepercommander/commands/pam_import/nsf_helpers.py @@ -395,6 +395,65 @@ def create_nsf_subfolder(params, folder_name: str, parent_uid: str = '', return folder_uid +_NSF_FOLDER_BATCH_LIMIT = 100 + + +def create_nsf_folders_batch(params, folder_specs: List[dict], *, + sync: bool = True, + command: str = 'pam') -> List[dict]: + """Create NSF folders via ``vault/folders/v3/add`` in chunks of 100. + + Each *folder_specs* entry is ``{'name': str, 'parent_uid': str|None}``. + Returns the batch API result list (same order as *folder_specs*). + Seeds local NSF caches after each successful create; optionally syncs once + at the end so callers avoid one round-trip per folder. + """ + from ...nested_share_folder.folder_api import create_folders_batch_v3 + + if not folder_specs: + return [] + + results: List[dict] = [] + for start in range(0, len(folder_specs), _NSF_FOLDER_BATCH_LIMIT): + chunk = folder_specs[start:start + _NSF_FOLDER_BATCH_LIMIT] + try: + chunk_results = create_folders_batch_v3(params, chunk) + except Exception as exc: + raise CommandError(command, f'NSF folder batch create failed: {exc}') from exc + + if len(chunk_results) != len(chunk): + raise CommandError( + command, + f'NSF folder batch returned {len(chunk_results)} results for {len(chunk)} folders', + ) + + for spec, result in zip(chunk, chunk_results): + if not result.get('success'): + name = spec.get('name') or '?' + raise CommandError( + command, + result.get('message') or f'Failed to create Nested Share Folder: {name}', + ) + folder_uid = result.get('folder_uid') + if not folder_uid: + raise CommandError( + command, + f"Nested Share Folder creation did not return UID: {spec.get('name')}", + ) + seed_nsf_folder_cache( + params, + folder_uid, + spec.get('name') or '', + spec.get('parent_uid') or None, + result.get('folder_key_unencrypted'), + ) + results.append(result) + + if sync: + sync_down_preserving_nsf_keys(params) + return results + + def extend_create_record(params, obj, folder_uid: str) -> Optional[str]: """Create a PAM import record in a classic or NSF folder.""" return obj.create_record(params, folder_uid) diff --git a/keepercommander/commands/pedm/pedm_admin.py b/keepercommander/commands/pedm/pedm_admin.py index 9534ad3e4..55008249c 100644 --- a/keepercommander/commands/pedm/pedm_admin.py +++ b/keepercommander/commands/pedm/pedm_admin.py @@ -2325,8 +2325,11 @@ def execute(self, context: KeeperParams, **kwargs) -> Any: if approval_type and approval_type != status.lower(): continue - account_info = [y[:30] for y in (f'{k}={v}' for k, v in approval.account_info.items())] - application_info = [y[:30] for y in (f'{k}={v}' for k, v in approval.application_info.items())] + account_info = [f'{k}={v}' for k, v in approval.account_info.items()] + application_info = [f'{k}={v}' for k, v in approval.application_info.items()] + if fmt != 'json': + account_info = [y[:30] for y in account_info] + application_info = [y[:30] for y in application_info] justification = approval.justification if fmt != 'json' and isinstance(justification, str): try: diff --git a/keepercommander/commands/tunnel/port_forward/tunnel_helpers.py b/keepercommander/commands/tunnel/port_forward/tunnel_helpers.py index 902939437..9d26d78c5 100644 --- a/keepercommander/commands/tunnel/port_forward/tunnel_helpers.py +++ b/keepercommander/commands/tunnel/port_forward/tunnel_helpers.py @@ -177,6 +177,51 @@ def print_above_keeper_prompt(msg): # VERIFY_SSL applies to WebSocket SSL only; HTTP uses params.ssl_verify. VERIFY_SSL = bool(os.environ.get("VERIFY_SSL", "TRUE") == "TRUE") + +def ephemeral_gateway_timeout_ms(is_ephemeral, default=GATEWAY_TIMEOUT): + """Gateway wait timeout (ms) for a tunnel-start offer. + + Ephemeral JIT account creation on the gateway takes 30-90s (mirrors the + connection path's bump in terminal_connection.py) — the normal + ``default`` (30s) is too tight for it. + """ + if not is_ephemeral: + return default + try: + return int(os.environ.get('PAM_GATEWAY_OFFER_TIMEOUT_EPHEMERAL_MS', '120000')) + except (TypeError, ValueError): + return 120000 + + +def credential_override_data_fields(credential_type, credential_data): + """Fields to merge into a tunnel-start offer's encrypted inner payload + for a --credential/-cr override. Only 'userSupplied' is recognized today + (the only value Commander's --credential flag sends); anything else + yields no fields, so plain tunnel starts are unaffected. + + private_key/passphrase are only included when present, so KeeperRDP + Proxy overrides (username/password-only) are unaffected. + """ + if credential_type != 'userSupplied' or not credential_data: + return {} + fields = { + 'username': credential_data.get('username', ''), + 'password': credential_data.get('password', ''), + } + if credential_data.get('private_key'): + fields['private_key'] = credential_data['private_key'] + if credential_data.get('passphrase'): + fields['passphrase'] = credential_data['passphrase'] + return fields + + +def credential_override_input_fields(credential_type): + """Fields to merge into a tunnel-start offer's outer (unencrypted) + gateway action inputs for a --credential/-cr override.""" + if credential_type != 'userSupplied': + return {} + return {'credentialType': 'userSupplied', 'allowSupplyUser': True} + # ICE candidate buffering - store until SDP answer is received # Global conversation key management for multiple concurrent tunnels @@ -312,7 +357,8 @@ class TunnelSession: """Container for tunnel session state organized by tube_id""" def __init__(self, tube_id, conversation_id, gateway_uid, symmetric_key, offer_sent=False, host=None, port=None, - record_title=None, record_uid=None, target_host=None, target_port=None): + record_title=None, record_uid=None, target_host=None, target_port=None, + proxy_kind=None, proxy_db_type=None): self.tube_id = tube_id self.conversation_id = conversation_id self.gateway_uid = gateway_uid @@ -324,6 +370,12 @@ def __init__(self, tube_id, conversation_id, gateway_uid, symmetric_key, self.record_uid = record_uid self.target_host = target_host self.target_port = target_port + # Which Keeper Proxy backend (if any) fronts this tunnel — 'db' | + # 'rdp' | 'ssh' | None. Drives the proxy-ready banner printed from + # the WebRTC 'connected' signal callback below, once the tunnel is + # actually up (not immediately after the offer is merely accepted). + self.proxy_kind = proxy_kind + self.proxy_db_type = proxy_db_type self.buffered_ice_candidates = [] self.creation_time = time.time() self.last_activity = time.time() @@ -331,6 +383,12 @@ def __init__(self, tube_id, conversation_id, gateway_uid, symmetric_key, self.websocket_thread = None self.websocket_ready_event = None self.websocket_stop_event = None + # Late-arrival channel for Gateway error payloads (is_ok=False). + # The -nti HTTP offer path waits on `gateway_error_event` briefly + # after an RRC_TIMEOUT, so a delayed "Unable to establish connection + # to remote host …" message can replace the generic timeout text. + self.gateway_error_message = None + self.gateway_error_event = threading.Event() # Optional attributes (set dynamically) # Note: signal_handler is set after TunnelSignalHandler is created self.signal_handler = None # type: ignore[assignment] @@ -1244,6 +1302,27 @@ async def handle_websocket_messages(websocket, tube_registry, timeout, stop_even logging.debug("WebSocket handler completed") +# Exact i18n keys GW's KeeperSSH Proxy setup path returns for its 3 fatal, +# post-eager-validation failures (KeeperSshProxyAuthError/TimeoutError/ +# ConfigError — see dr-controller's WebRTCSessionAction.setup_keeperssh_proxy_for_tunnel). +# Deliberately excludes the generic `..._keeperssh_proxy_error` code (missing +# module / missing credentials / unexpected error) — those are covered by +# earlier, synchronous validation and don't need this async fallback. +# +# NOTE: this is a match on the *untranslated* i18n key, not a stable error +# code contract — GW's gettext-based i18n falls back to returning the raw +# key verbatim only because no locale catalog (not even the "us_FB" English +# fallback) has real text for these new keys yet. Once one does, `data` +# becomes a translated sentence with no key substring in it and this match +# silently stops firing (reverting to today's leaked-tube behavior for this +# one case, not a new failure mode). +_SSH_PROXY_FATAL_SETUP_ERROR_CODES = frozenset({ + 'gateway_webrtcaction_keeperssh_proxy_auth_failed', + 'gateway_webrtcaction_keeperssh_proxy_timeout', + 'gateway_webrtcaction_keeperssh_proxy_config_error', +}) + + def route_message_to_rust(response_item, tube_registry): """Route a single message to Rust - decrypt it first using the conversation's key""" try: @@ -1542,9 +1621,55 @@ def route_message_to_rust(response_item, tube_registry): elif not payload_data.get('is_ok', True): # Gateway returned an explicit error (is_ok=False) — log the message and move on. # This includes auth failures (401 on get_leafs), overload responses, etc. + gateway_error_text = payload_data.get('data', 'unknown error') logging.error( - f"Gateway error for {conversation_id}: {payload_data.get('data', 'unknown error')}" + f"Gateway error for {conversation_id}: {gateway_error_text}" ) + try: + err_tube_id = tube_registry.tube_id_from_connection_id(conversation_id) + if not err_tube_id: + url_safe_cid = conversation_id.replace('+', '-').replace('/', '_').rstrip('=') + err_tube_id = tube_registry.tube_id_from_connection_id(url_safe_cid) + if err_tube_id: + err_session = get_tunnel_session(err_tube_id) + if err_session is not None: + err_session.gateway_error_message = gateway_error_text + err_session.gateway_error_event.set() + + # KeeperSSH Proxy setup can fail asynchronously — Rust's + # real upstream auth/connect attempt takes a real network + # round trip, so the error can arrive after the synchronous + # connect-retry window has already given up and treated the + # tube as established (nothing else ever reads + # gateway_error_message/_event again after that point, so + # the tube would otherwise leak open forever). Close it here + # instead, treating it as a failed tunnel open — but only + # for the 3 known-fatal SSH Proxy setup codes, and only + # within the same window as the longest legitimate startup + # delay (ephemeral JIT account creation), so a plain + # (non-proxied) SSH tunnel, a different proxy backend, or an + # already-established/long-running KeeperSSH Proxy tunnel is + # never torn down by this. + if ( + gateway_error_text in _SSH_PROXY_FATAL_SETUP_ERROR_CODES + and err_session is not None + and (time.time() - err_session.creation_time) + <= (ephemeral_gateway_timeout_ms(True) / 1000.0) + ): + logging.error( + f"KeeperSSH Proxy failed to open for {conversation_id} " + f"({gateway_error_text}); closing tube {err_tube_id}" + ) + try: + tube_registry.close_tube(err_tube_id, reason=CloseConnectionReasons.Normal) + except Exception as close_exc: + logging.debug(f"Failed to close tube {err_tube_id} after gateway error: {close_exc}") + try: + unregister_tunnel_session(err_tube_id) + except Exception as unreg_exc: + logging.debug(f"Failed to unregister tunnel session {err_tube_id} after gateway error: {unreg_exc}") + except Exception as stash_exc: + logging.debug(f"Failed to stash gateway error on session: {stash_exc}") elif payload_data.get('data', '') == '': logging.debug("Empty data field an acknowledgment, no action needed") elif payload_data.get('data') and "ice candidate added" in payload_data.get('data').lower(): @@ -1642,6 +1767,55 @@ def __init__(self, conversation_id, record_uid): self.conversation_id = conversation_id +def _print_proxy_ready_banner(session): + """Print the Keeper Proxy (KeeperDB/KeeperRDP/KeeperSSH) readiness + banner for `session`. + + Called only from TunnelSignalHandler's WebRTC 'connected' callback, + never right after start_rust_tunnel returns — for a proxy tunnel, that + return only means the WebRTC offer was accepted, not that the gateway + has actually spawned the proxy and authenticated against the real + target yet. That can still fail (bad credentials, upstream timeout) + well after the offer is accepted, and a valid SDP answer is never even + generated in that case, so the 'connected' state this banner is gated + on never fires either — printing here means the banner only ever shows + for a tunnel that is actually up. + """ + host, port = session.host, session.port + if session.proxy_kind == 'db': + db_type = session.proxy_db_type + suffix = f' ({db_type})' if db_type else '' + logging.info(f"\n{bcolors.OKGREEN}KeeperDB Proxy ready{suffix}{bcolors.ENDC}") + logging.info(f" Listening: {host}:{port}") + if db_type == 'mysql': + logging.info(f" Connect: mysql -h {host} -P {port} -u -p") + elif db_type == 'postgresql': + logging.info(f" Connect: psql -h {host} -p {port} -U ") + elif db_type == 'mssql': + logging.info(f" Connect: sqlcmd -S {host},{port} -U -P ") + else: + logging.info(f" Connect: use your database client to connect to {host}:{port}") + logging.info(f"{bcolors.OKBLUE} Note: when your DB client prompts for credentials you may " + f"supply any value — the proxy will substitute the credentials configured in your " + f"Keeper vault.{bcolors.ENDC}") + elif session.proxy_kind == 'rdp': + logging.info(f"\n{bcolors.OKGREEN}KeeperRDP Proxy ready{bcolors.ENDC}") + logging.info(f" Listening: {host}:{port}") + logging.info(f" Connect: mstsc /v:{host}:{port} " + f"(or wfreerdp /v:{host}:{port} /sec:tls /cert:ignore)") + logging.info(f"{bcolors.OKBLUE} Note: when your RDP client prompts for credentials supply any " + f"value (or the per-session NLA token if NLA is enabled). The proxy substitutes the " + f"credentials configured in your Keeper vault before forwarding to the target." + f"{bcolors.ENDC}") + elif session.proxy_kind == 'ssh': + logging.info(f"\n{bcolors.OKGREEN}KeeperSSH Proxy ready{bcolors.ENDC}") + logging.info(f" Listening: {host}:{port}") + logging.info(f" Connect: ssh -p {port} @{host}") + logging.info(f"{bcolors.OKBLUE} Note: when your SSH client prompts for credentials supply any " + f"value. The proxy substitutes the credentials configured in your Keeper vault before " + f"forwarding to the target.{bcolors.ENDC}") + + # Callback handler class for WebRTC signals class TunnelSignalHandler: """ @@ -1750,7 +1924,15 @@ def _signal_from_rust_inner(self, response: dict): # Get tunnel session for record details if session and not self.silent: - logging.info(f"\n{bcolors.OKGREEN}Connection established successfully.{bcolors.ENDC}") + # Proxy-ready banner prints here, right before "Connection + # established successfully" with no delimiter (e.g. shell + # prompt) between them — see _print_proxy_ready_banner's + # docstring for why this can't be printed any earlier. + if session.proxy_kind: + _print_proxy_ready_banner(session) + logging.info(f"{bcolors.OKGREEN}Connection established successfully.{bcolors.ENDC}") + else: + logging.info(f"\n{bcolors.OKGREEN}Connection established successfully.{bcolors.ENDC}") # Display record title if available if session.record_title: @@ -2318,7 +2500,7 @@ def cleanup(self): logging.debug("TunnelSignalHandler cleaned up") def start_rust_tunnel(params, record_uid, gateway_uid, host, port, - seed, target_host, target_port, socks, trickle_ice=True, record_title=None, allow_supply_host=False, two_factor_value=None, kind='start', probe_duration=30, probe_turn_only=False, probe_stun_only=False): + seed, target_host, target_port, socks, trickle_ice=True, record_title=None, allow_supply_host=False, two_factor_value=None, kind='start', probe_duration=30, probe_turn_only=False, probe_stun_only=False, credential_type=None, credential_data=None, is_ephemeral=False, proxy_kind=None, proxy_db_type=None): """ Start a tunnel using Rust WebRTC with trickle ICE via HTTP POST and WebSocket responses. @@ -2461,7 +2643,9 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, record_title=record_title, record_uid=record_uid, target_host=target_host, - target_port=target_port + target_port=target_port, + proxy_kind=proxy_kind, + proxy_db_type=proxy_db_type, ) # Register the temporary session so ICE candidates can be buffered immediately @@ -2636,6 +2820,20 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, } logging.debug(f"Including user-supplied host in payload: {target_host}:{target_port}") + # KeeperRDP Proxy and KeeperDB Proxy both auto-route on the + # gateway side from the record's `allowKeeper{DB,RDP}Proxy` + # setting alone, so no client-side opt-in flag is included in + # the WebRTC payload here. The Commander-side `--proxy` flag is + # used for validators (record-type check, `allowKeeperXxxProxy` + # presence check, launch-credential preflight) and to print the + # right banner; it does not need to round-trip to the gateway. + # + # credential_type/credential_data are the exception: an explicit + # --credential override DOES need to round-trip, since it's telling + # the gateway to use inline username/password instead of its own + # ephemeral/linked resolution. + data.update(credential_override_data_fields(credential_type, credential_data)) + string_data = json.dumps(data) bytes_data = string_to_bytes(string_data) encrypted_data = tunnel_encrypt(symmetric_key, bytes_data) @@ -2674,6 +2872,7 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, } if two_factor_value: inputs['twoFactorValue'] = two_factor_value + inputs.update(credential_override_input_fields(credential_type)) router_response = router_send_action_to_gateway( params=params, @@ -2684,7 +2883,7 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, ), message_type=pam_pb2.CMT_CONNECT, is_streaming=trickle_ice, - gateway_timeout=GATEWAY_TIMEOUT, + gateway_timeout=ephemeral_gateway_timeout_ms(is_ephemeral), **offer_kwargs ) @@ -2699,6 +2898,7 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, except Exception as e: error_msg = str(e) is_bad_state = "RRC_BAD_STATE" in error_msg + is_timeout = "RRC_TIMEOUT" in error_msg is_last_attempt = (attempt == max_retries) if is_bad_state and not is_last_attempt: @@ -2712,7 +2912,21 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, logging.error(f"RRC_BAD_STATE persists after {max_retries} retries") logging.error("This may indicate network issues or backend problems") - logging.error(f"Failed to send offer via HTTP: {error_msg}") + # On RRC_TIMEOUT, give the Gateway a brief grace period to + # deliver a real error payload via WebSocket (e.g. "Unable + # to establish connection to remote host …"). Without this + # the user only sees the generic HTTP timeout even when the + # gateway already knows why upstream is unreachable. + final_error = error_msg + if is_timeout and not tunnel_session.gateway_error_event.is_set(): + logging.debug( + "RRC_TIMEOUT — waiting up to 8s for delayed Gateway error payload" + ) + tunnel_session.gateway_error_event.wait(timeout=8.0) + if tunnel_session.gateway_error_message: + final_error = tunnel_session.gateway_error_message + + logging.error(f"Failed to send offer via HTTP: {final_error}") # Cleanup on final failure logging.debug(f"Cleaning up failed tunnel {commander_tube_id}") @@ -2734,7 +2948,7 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, signal_handler.cleanup() unregister_tunnel_session(commander_tube_id) - return {"success": False, "error": f"Failed to send offer via HTTP: {e}"} + return {"success": False, "error": f"Failed to start tunnel: {final_error}"} # Continue with the rest of the flow after successful offer send # Trickle ICE: Response comes via WebSocket (HTTP response is empty) @@ -2753,6 +2967,26 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, payload_json = json.loads(payload_str) logging.debug(f"Non-trickle ICE: Parsed payload JSON, keys: {payload_json.keys()}") + # Gateway-side error: the HTTP response carries an + # is_ok=False envelope whose 'data' field is a plain- + # text reason rather than an encrypted SDP answer. + # Surface it with the same one-line wording the + # trickle path uses (route_message_to_rust) and bail + # out — otherwise the code below decrypts garbage, + # raises a TypeError in json.loads(None), and the + # banner prints anyway. + if not payload_json.get('is_ok', True): + gateway_error_text = payload_json.get('data', 'unknown error') + logging.error( + f"Gateway error for {conversation_id_original}: {gateway_error_text}" + ) + if tunnel_session.websocket_stop_event and tunnel_session.websocket_thread: + tunnel_session.websocket_stop_event.set() + tunnel_session.websocket_thread.join(timeout=2.0) + signal_handler.cleanup() + unregister_tunnel_session(commander_tube_id) + return {"success": False, "error": f"Failed to start tunnel: {gateway_error_text}"} + encrypted_answer = payload_json.get('data') if encrypted_answer: # Decrypt the answer using the tunnel's symmetric key @@ -2773,8 +3007,6 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, logging.error(f"Non-trickle ICE: No 'payload' field in gateway response: {gateway_payload}") except Exception as e: logging.error(f"Non-trickle ICE: Failed to process HTTP response: {e}") - import traceback - logging.error(f"Traceback: {traceback.format_exc()}") # Send any buffered ICE candidates that arrived before offer was sent (trickle ICE only) if trickle_ice and tunnel_session.buffered_ice_candidates: @@ -2811,6 +3043,7 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, "conversation_id": conversation_id_original, # Use original, not base64 encoded "tube_registry": tube_registry, "status": "connecting", # Indicates async connection in progress + "local_host": tunnel_session.host, # Actual bound host (usually 127.0.0.1) "local_port": tunnel_session.port, # Actual bound port (may differ from requested) } diff --git a/keepercommander/commands/tunnel_and_connections.py b/keepercommander/commands/tunnel_and_connections.py index f818ef8f7..fe8440e6f 100644 --- a/keepercommander/commands/tunnel_and_connections.py +++ b/keepercommander/commands/tunnel_and_connections.py @@ -11,14 +11,13 @@ import argparse import datetime -import http.client import json import logging import os import platform +import requests import signal import socket -import ssl import struct import subprocess import sys @@ -29,15 +28,17 @@ from .base import Command, GroupCommand, dump_report_data, RecordMixin from .tunnel.port_forward.TunnelGraph import TunnelDAG -from .tunnel.port_forward.tunnel_helpers import find_open_port, get_config_uid, get_config_uid_via_pam_link, \ - get_keeper_tokens, \ +from .tunnel.port_forward.tunnel_helpers import find_open_port, is_port_open, get_config_uid, \ + get_config_uid_via_pam_link, get_keeper_tokens, \ get_or_create_tube_registry, get_gateway_uid_from_record, resolve_record, resolve_pam_config, resolve_folder, \ remove_field, start_rust_tunnel, get_tunnel_session, unregister_tunnel_session, CloseConnectionReasons, \ wait_for_tunnel_connection, create_rust_webrtc_settings, \ print_above_keeper_prompt from .pam.router_helper import get_dag_leafs -from .pam.vault_target import update_pam_record, reload_pam_record_if_nsf_updated +from .pam.vault_target import update_pam_record, reload_pam_record_if_nsf_updated, records_in_folder +from .pam_import.keeper_ai_settings import get_resource_jit_settings from .pam_import.nsf_helpers import sync_down_preserving_nsf_keys +from .pam_launch.terminal_connection import _extract_user_record_credentials from .tunnel_registry import ( PARENT_GRACE_SECONDS, is_pid_alive, @@ -51,7 +52,7 @@ from ..error import CommandError import json from ..params import LAST_RECORD_UID -from ..subfolder import find_folders +from ..subfolder import find_folders, try_resolve_path from ..utils import value_to_boolean from ..constants import get_relay_host, get_router_host, get_keeper_server_hostname @@ -90,6 +91,128 @@ def _coerce_settings_subdicts(entry, *keys): return changed +def _resource_has_ephemeral_jit(params, record_uid): + """True if the resource's jit_settings.createEphemeral is enabled on the DAG. + + Non-fatal on any read failure (unconfigured/unreadable jit_settings just + means the record isn't JIT-configured, which is the normal case). + """ + try: + jit = get_resource_jit_settings(params, record_uid) + except Exception as e: + logging.debug(f"Could not read jit_settings for {record_uid}: {e}") + return False + return bool(jit and jit.get('createEphemeral')) + + +def _classify_proxy_target(record_uid, record_type, protocol, is_keeper_proxy, launch_credential): + """Classify a `pam tunnel start` target into a proxy backend, given the + record's type/protocol and which of --proxy / --credential were passed. + + Pure decision logic (no params/network access) so it's directly + unit-testable. Returns (is_keeperdb_proxy, is_keeperrdp_proxy, + is_keeperssh_proxy, error) — ``error`` is a message to print and abort + on, or None to proceed. + """ + if record_type == 'pamDatabase': + is_keeperdb_proxy, is_keeperrdp_proxy, is_keeperssh_proxy = True, False, False + elif record_type == 'pamMachine' and protocol == 'rdp': + is_keeperdb_proxy, is_keeperrdp_proxy, is_keeperssh_proxy = False, True, False + elif record_type == 'pamMachine' and protocol == 'ssh': + is_keeperdb_proxy, is_keeperrdp_proxy, is_keeperssh_proxy = False, False, True + else: + detail = f' (protocol="{protocol}")' if record_type == 'pamMachine' else '' + if launch_credential and not is_keeper_proxy: + return False, False, False, "--credential requires a proxied tunnel; plain tunnels don't authenticate." + return False, False, False, ( + f'--proxy is supported on pamDatabase or pamMachine+RDP/SSH records. ' + f'Record {record_uid} is of type "{record_type}"{detail}.' + ) + + if launch_credential and is_keeperdb_proxy: + return False, False, False, "--credential is not yet supported for KeeperDB Proxy tunnels." + + return is_keeperdb_proxy, is_keeperrdp_proxy, is_keeperssh_proxy, None + + +def _detect_proxy_backend(record_type, protocol, pam_settings_value): + """Auto-detect which Keeper Proxy backend (if any) actually applies to + this record, purely from its type/protocol/allowKeeper*Proxy flags — + independent of whether --proxy/--credential were passed on this + invocation. The gateway auto-routes from these flags alone regardless of + any client-side flag, so a plain (linked-credential/ephemeral-JIT) + tunnel start is just as much a proxy tunnel as one started with + --proxy/--credential; this drives the post-start proxy-ready banner for + all of them, not just the --proxy/--credential case. + + Pure decision logic (no params/network access) so it's directly + unit-testable. Returns (is_keeperdb_proxy, is_keeperrdp_proxy, + is_keeperssh_proxy) — at most one True, since the three backends are + mutually exclusive by record type/protocol. + """ + if not isinstance(pam_settings_value, dict): + return False, False, False + if record_type == 'pamDatabase': + allow_kdb = bool( + (pam_settings_value.get('portForward') or {}).get('allowKeeperDBProxy') + or (pam_settings_value.get('connection') or {}).get('allowKeeperDBProxy') + ) + return allow_kdb, False, False + if record_type == 'pamMachine' and protocol == 'rdp': + allow_krdp = bool((pam_settings_value.get('portForward') or {}).get('allowKeeperRDPProxy')) + return False, allow_krdp, False + if record_type == 'pamMachine' and protocol == 'ssh': + allow_kssh = bool((pam_settings_value.get('portForward') or {}).get('allowKeeperSSHProxy')) + return False, False, allow_kssh + return False, False, False + + +def _pam_allow_supply_user(pam_settings_value): + """Read pamSettings.connection.allowSupplyUser — the checkbox that gates + --credential's userSupplied override (and, on the connections side, the + equivalent -cr/--credential flag).""" + if not isinstance(pam_settings_value, dict): + return False + return bool((pam_settings_value.get('connection') or {}).get('allowSupplyUser')) + + +def _resolve_credential_record(params, token): + """Resolve a --credential/-cr token (UID, path, or title) to a record UID. + + Lightweight resolver for `pam tunnel start --credential`: supports UID and + path/title lookup across any record type. Unlike `pam launch`'s resolver, + it has no substring fallback and no interactive multi-match picker — + ambiguous or unmatched input simply returns None so the caller can print a + single clear error. + """ + if not token: + return None + token = token.strip() + + if token in params.record_cache: + return token + + rs = try_resolve_path(params, token) + if rs is not None: + folder, name = rs + if folder is not None and name: + folder_uid = folder.uid or '' + for uid in (records_in_folder(params, folder_uid) or []): + record = vault.KeeperRecord.load(params, uid) + if record and record.title and record.title.lower() == name.lower(): + return uid + + token_lower = token.lower() + matches = [] + for uid in params.record_cache: + record = vault.KeeperRecord.load(params, uid) + if record and record.title and record.title.lower() == token_lower: + matches.append(uid) + if len(matches) == 1: + return matches[0] + return None + + # Group Commands class PAMTunnelCommand(GroupCommand): @@ -391,6 +514,13 @@ class PAMTunnelEditCommand(Command): pam_cmd_parser.add_argument('--keeper-db-proxy', '-kdbp', required=False, dest='keeper_db_proxy', choices=['on', 'off', 'default'], help='Enable/disable Keeper Database Proxy for pamDatabase records (on/off/default)') + pam_cmd_parser.add_argument('--keeper-proxy', '-kp', required=False, dest='keeper_proxy', + choices=['on', 'off', 'default'], + help='Enable/disable Keeper Proxy, auto-selecting the backend from the record\'s ' + 'connection protocol: KeeperDB Proxy for database protocols ' + '(pamDatabase, or pamMachine set to a database protocol), KeeperRDP Proxy ' + 'for RDP (pamMachine). SSH is not yet supported. Records/protocols that ' + 'don\'t map to a proxy backend are ignored with a warning (on/off/default)') def get_parser(self): return PAMTunnelEditCommand.pam_cmd_parser @@ -567,12 +697,16 @@ def execute(self, params, **kwargs): f'{bcolors.FAIL}--keeper-db-proxy is only supported for pamDatabase records. ' f'Record "{record_name}" is of type "{record_type}".{bcolors.ENDC}') if keeper_db_proxy == 'on' and not tmp_dag.check_if_resource_has_launch_credential(record_uid): - raise CommandError('', - f'{bcolors.FAIL}No Launch Credentials assigned to record "{record_uid}". ' - f'Please assign launch credentials to the record before enabling ' - f'the database proxy.\n' - f'Use: {bcolors.OKBLUE}pam connection edit ' - f'--launch-user (-lu) {bcolors.ENDC}') + if not _resource_has_ephemeral_jit(params, record_uid): + raise CommandError('', + f'{bcolors.FAIL}No Launch Credentials assigned to record "{record_uid}". ' + f'Please assign launch credentials to the record before enabling ' + f'the database proxy.\n' + f'Use: {bcolors.OKBLUE}pam connection edit ' + f'--launch-user (-lu) {bcolors.ENDC}') + logging.debug(f"No linked launch credential found for record {record_uid}; " + f"ephemeral JIT is enabled, proceeding without a static " + f"launch credential") if not pam_settings: pam_settings = vault.TypedField.new_field('pamSettings', {"connection": {}, "portForward": {}}, "") record.custom.append(pam_settings) @@ -591,6 +725,71 @@ def execute(self, params, **kwargs): pam_settings.value[0]["connection"].pop('allowKeeperDBProxy', None) dirty = True + # Handle --keeper-proxy: generalized on/off/default that auto-selects the + # backend (KeeperDB/KeeperRDP/KeeperSSH) from the record's connection + # protocol instead of requiring a protocol-specific flag. Missing + # option -> no-op, no validation at all. + keeper_proxy = kwargs.get('keeper_proxy') + if keeper_proxy: + protocol = '' + if pam_settings and pam_settings.value and isinstance(pam_settings.value[0], dict): + protocol = (pam_settings.value[0].get('connection') or {}).get('protocol') or '' + protocol = str(protocol).strip().lower() + + proxy_key = None + proxy_bucket = None + if record_type == 'pamDatabase' or protocol in PAMConnectionEditCommand.db_protocols: + proxy_key = 'allowKeeperDBProxy' + proxy_bucket = 'connection' + elif protocol == 'rdp': + proxy_key = 'allowKeeperRDPProxy' + proxy_bucket = 'portForward' + elif protocol == 'ssh': + proxy_key = 'allowKeeperSSHProxy' + proxy_bucket = 'portForward' + else: + print(f"{bcolors.WARNING}--keeper-proxy has no effect for record {record_uid} " + f"(type=\"{record_type}\", protocol=\"{protocol}\"). KeeperDB Proxy applies " + f"only to database records/protocols, KeeperRDP Proxy only to RDP records, " + f"and KeeperSSH Proxy only to SSH records.{bcolors.ENDC}") + + if proxy_key: + if keeper_proxy == 'default': + # Pure deletion: never create pamSettings or its connection/portForward + # buckets just to immediately leave them empty. No-op (no validation, + # no error) if pamSettings, the bucket, or the key isn't there at all. + if pam_settings and pam_settings.value and isinstance(pam_settings.value[0], dict): + bucket = pam_settings.value[0].get(proxy_bucket) + if isinstance(bucket, dict) and proxy_key in bucket: + bucket.pop(proxy_key, None) + dirty = True + else: + if keeper_proxy == 'on' and not tmp_dag.check_if_resource_has_launch_credential(record_uid): + if not _resource_has_ephemeral_jit(params, record_uid): + raise CommandError('', + f'{bcolors.FAIL}No Launch Credentials assigned to record "{record_uid}". ' + f'Please assign launch credentials to the record before enabling ' + f'the proxy.\n' + f'Use: {bcolors.OKBLUE}pam connection edit ' + f'--launch-user (-lu) {bcolors.ENDC}') + logging.debug(f"No linked launch credential found for record {record_uid}; " + f"ephemeral JIT is enabled, proceeding without a static " + f"launch credential") + if not pam_settings: + pam_settings = vault.TypedField.new_field('pamSettings', {"connection": {}, "portForward": {}}, "") + record.custom.append(pam_settings) + if not pam_settings.value: + pam_settings.value.append({"connection": {}, "portForward": {}}) + if not isinstance(pam_settings.value[0], dict): + pam_settings.value[0] = {"connection": {}, "portForward": {}} + if not isinstance(pam_settings.value[0].get(proxy_bucket), dict): + pam_settings.value[0][proxy_bucket] = {} + current_value = pam_settings.value[0][proxy_bucket].get(proxy_key) + target = (keeper_proxy == 'on') + if current_value is not target: + pam_settings.value[0][proxy_bucket][proxy_key] = target + dirty = True + if dirty: tmp_dag.set_resource_allowed(resource_uid=record_uid, tunneling=_tunneling, allowed_settings_name=allowed_settings_name) was_nsf = update_pam_record(params, record, command='pam tunnel edit') @@ -629,10 +828,16 @@ class PAMTunnelStartCommand(Command): help='Disable trickle ICE for WebRTC connections. By default, trickle ICE is enabled ' 'for real-time candidate exchange.') pam_cmd_parser.add_argument('--proxy', '-px', required=False, dest='proxy', action='store_true', - help='Activate KeeperDB Proxy: the gateway substitutes credentials ' - 'from your Keeper vault when the local client connects to the tunnel.') - # TODO(rdp-proxy): once RDP Proxy support lands on pamMachine, generalize --proxy - # to or auto-detect from the record type). For now, --proxy is KeeperDB-only. + help='Activate Keeper Proxy (KeeperDB for pamDatabase, KeeperRDP for ' + 'pamMachine + RDP, KeeperSSH for pamMachine + SSH): the gateway ' + 'substitutes credentials from your Keeper vault when the local ' + 'client connects to the tunnel.') + pam_cmd_parser.add_argument('--credential', '-cr', required=False, dest='launch_credential', type=str, + help='Record (UID, path, or title) to use for KeeperRDP/KeeperSSH Proxy ' + 'tunnel credentials, overriding the resource\'s linked launch ' + 'credential or ephemeral-JIT setting. Requires "Allow users to select ' + 'credentials from their vault" (allowSupplyUser) on the record. Not ' + 'supported for plain (non-proxied) tunnels.') pam_cmd_parser.add_argument('--reason', '-r', required=False, dest='workflow_reason', type=str, help='Justification text for workflow access request. Used when the record\'s ' 'workflow requires a reason; non-interactive equivalent of the inline prompt.') @@ -705,21 +910,13 @@ def _resolve_database_type(record, pam_settings_value): return None @staticmethod - def _print_keeperdb_proxy_banner(host, port, db_type): - suffix = f' ({db_type})' if db_type else '' - print(f"\n{bcolors.OKGREEN}KeeperDB Proxy ready{suffix}{bcolors.ENDC}") - print(f" Listening: {host}:{port}") - if db_type == 'mysql': - print(f" Connect: mysql -h {host} -P {port} -u -p") - elif db_type == 'postgresql': - print(f" Connect: psql -h {host} -p {port} -U ") - elif db_type == 'mssql': - print(f" Connect: sqlcmd -S {host},{port} -U -P ") - else: - print(f" Connect: use your database client to connect to {host}:{port}") - print(f"{bcolors.OKBLUE} Note: when your DB client prompts for credentials you may " - f"supply any value — the proxy will substitute the credentials configured in your " - f"Keeper vault.{bcolors.ENDC}") + def _resolve_connection_protocol(pam_settings_value): + # Lower-cased connection protocol from the record's pamSettings + # ('rdp' | 'ssh' | 'vnc' | 'telnet' | ''). pamMachine uses this to + # pick the right Gateway proxy backend. + if isinstance(pam_settings_value, dict): + return ((pam_settings_value.get('connection') or {}).get('protocol') or '').strip().lower() + return '' def execute(self, params, **kwargs): # Python version validation (same as before) @@ -746,6 +943,27 @@ def execute(self, params, **kwargs): port = kwargs.get('port') no_trickle_ice = kwargs.get('no_trickle_ice', False) + # Port selection policy (matches the keeper-pam-webrtc-rs local + # listener's SO_EXCLUSIVEADDRUSE behavior on Windows): + # * user passed `--port N` (any N, including 49152) → strict: + # ask Rust for exactly N; if the bind fails (port already + # in use, permission denied, etc.) surface the error and + # don't silently pick a different port. The intent of + # `--port` is "I want exactly this port; if you can't give + # it to me, tell me." + # * no `--port` given → prefer `LEGACY_DEFAULT_TUNNEL_PORT` + # (49152) for backward-compatibility with docs / muscle + # memory that hard-code the connect string. Pre-probe with + # `is_port_open` (which is authoritative on Windows now + # that Rust uses SO_EXCLUSIVEADDRUSE) to avoid the noisy + # "address in use" error path in the common case where + # 49152 is already taken. If the probe says it's free we + # still keep `allow_ephemeral_fallback=True` so the rare + # TOCTOU race (something grabs 49152 between the probe and + # the Rust bind) falls back silently to an ephemeral port + # instead of failing the whole start. + LEGACY_DEFAULT_TUNNEL_PORT = 49152 + allow_ephemeral_fallback = False if port is not None and port > 0: try: port = find_open_port(tried_ports=[], preferred_port=port, host=host) @@ -753,10 +971,12 @@ def execute(self, params, **kwargs): print(f"{bcolors.FAIL}{e}{bcolors.ENDC}") return else: - port = find_open_port(tried_ports=[], host=host) - if port is None: - print(f"{bcolors.FAIL}Could not find open port to use for tunnel{bcolors.ENDC}") - return + probe_host = host or '127.0.0.1' + if is_port_open(probe_host, LEGACY_DEFAULT_TUNNEL_PORT): + port = LEGACY_DEFAULT_TUNNEL_PORT + allow_ephemeral_fallback = True + else: + port = 0 # Sync and validate record api.sync_down(params) @@ -831,48 +1051,138 @@ def execute(self, params, **kwargs): pam_settings_value = pam_settings.get_default_value() if pam_settings else {} allow_supply_host = pam_settings_value.get('allowSupplyHost', False) if isinstance(pam_settings_value, dict) else False - # --proxy: KeeperDB Proxy mode (gateway substitutes credentials from vault). - # This is a Commander-side validator/declaration; the gateway currently - # auto-routes pamDatabase + allowKeeperDBProxy to the proxy regardless of - # any client-side flag (see is_keeperdb_proxy_tunnel in dr-controller's - # tunnel_helpers.py and _build_protocol_settings in WebRTCSessionAction.py). - # Requiring no-`--proxy` to mean "raw TCP tunnel to remote host" depends on - # a future gateway change to honor a client-side opt-in flag; until that - # lands, omitting --proxy will still proxy if the record allows it. - is_keeperdb_proxy = bool(kwargs.get('proxy')) - db_type_for_banner = None - if is_keeperdb_proxy: - record_type = record.record_type - # TODO(rdp-proxy): once RDP Proxy support lands, also accept - # 'pamMachine' here and dispatch by record type. - if record_type != 'pamDatabase': - print(f"{bcolors.FAIL}--proxy is only supported on pamDatabase records. " - f"Record {record_uid} is of type \"{record_type}\".{bcolors.ENDC}") - return - allow_kdb = isinstance(pam_settings_value, dict) and bool( - (pam_settings_value.get('portForward') or {}).get('allowKeeperDBProxy') - or (pam_settings_value.get('connection') or {}).get('allowKeeperDBProxy') - ) - if not allow_kdb: - print(f"{bcolors.FAIL}KeeperDB Proxy is not enabled for record {record_uid}.{bcolors.ENDC}") - print(f"{bcolors.WARNING}Enable it with " - f"{bcolors.OKBLUE}'pam tunnel edit {record_uid} --keeper-db-proxy on'" - f"{bcolors.ENDC}") - return - # Mirror the launch-credential pre-flight from PAMTunnelEditCommand - # (--keeper-db-proxy on path) so the failure message and timing are - # identical between edit and start. - _est, _ett, _tk = get_keeper_tokens(params) - _existing_cfg = get_config_uid(params, _est, _ett, record_uid) - _proxy_dag = TunnelDAG(params, _est, _ett, _existing_cfg, transmission_key=_tk) - if not _proxy_dag.check_if_resource_has_launch_credential(record_uid): - print(f"{bcolors.FAIL}No Launch Credentials assigned to record \"{record_uid}\". " - f"Please assign launch credentials before using --proxy.{bcolors.ENDC}") - print(f"{bcolors.WARNING}Use: " - f"{bcolors.OKBLUE}pam connection edit --launch-user (-lu) " - f"{bcolors.ENDC}") + # Auto-detect which Keeper Proxy backend (if any) actually applies to + # this record — purely from its type/protocol/allowKeeper*Proxy + # flags, independent of whether --proxy/--credential were passed on + # this invocation. The gateway auto-routes from these flags alone + # regardless of any client-side flag, so a plain (linked-credential/ + # ephemeral-JIT) tunnel start on a proxy-enabled record is just as + # much a proxy tunnel as one started with --proxy/--credential, and + # must show the same post-start proxy-ready banner. + record_type = record.record_type + protocol = self._resolve_connection_protocol(pam_settings_value) + is_keeperdb_proxy, is_keeperrdp_proxy, is_keeperssh_proxy = _detect_proxy_backend( + record_type, protocol, pam_settings_value) + db_type_for_banner = self._resolve_database_type(record, pam_settings_value) if is_keeperdb_proxy else None + + # --proxy: purely a client-side declaration that drives the extra + # validators below (record-type mismatch errors, launch-credential + # preflight) — actual backend detection (and the banner) above + # already happens unconditionally, since the gateway auto-routes + # from the record's own flags regardless of this flag. + is_keeper_proxy = bool(kwargs.get('proxy')) + # --credential/-cr: explicit credential override for a KeeperRDP/ + # KeeperSSH Proxy tunnel. Valid only when the target actually + # resolves to one of those proxy tunnels (allowKeeperRDPProxy / + # allowKeeperSSHProxy on a pamMachine record) — those are the only + # backends that authenticate against the target, so they're the only + # ones an explicit credential can apply to. Detected independent of + # --proxy: the gateway auto-routes from the record's own + # allowKeeper{RDP,SSH}Proxy setting regardless of whether --proxy was + # passed on this invocation (same as --target-host/allowSupplyHost + # today), so --credential alone is enough to opt in. + launch_credential = kwargs.get('launch_credential') + credential_type_override = None + credential_data_override = None + if is_keeper_proxy or launch_credential: + _cls_is_db, _cls_is_rdp, _cls_is_ssh, proxy_target_error = _classify_proxy_target( + record_uid, record_type, protocol, is_keeper_proxy, launch_credential) + if proxy_target_error: + print(f"{bcolors.FAIL}{proxy_target_error}{bcolors.ENDC}") return - db_type_for_banner = self._resolve_database_type(record, pam_settings_value) + + if _cls_is_db: + if not is_keeperdb_proxy: + print(f"{bcolors.FAIL}KeeperDB Proxy is not enabled for record {record_uid}.{bcolors.ENDC}") + print(f"{bcolors.WARNING}Enable it with " + f"{bcolors.OKBLUE}'pam tunnel edit {record_uid} --keeper-db-proxy on'" + f"{bcolors.ENDC}") + return + # Mirror the launch-credential pre-flight from PAMTunnelEditCommand + # (--keeper-db-proxy on path) so the failure message and timing are + # identical between edit and start. + _est, _ett, _tk = get_keeper_tokens(params) + _existing_cfg = get_config_uid(params, _est, _ett, record_uid) + _proxy_dag = TunnelDAG(params, _est, _ett, _existing_cfg, transmission_key=_tk) + if not _proxy_dag.check_if_resource_has_launch_credential(record_uid): + if not _resource_has_ephemeral_jit(params, record_uid): + print(f"{bcolors.FAIL}No Launch Credentials assigned to record \"{record_uid}\". " + f"Please assign launch credentials before using --proxy.{bcolors.ENDC}") + print(f"{bcolors.WARNING}Use: " + f"{bcolors.OKBLUE}pam connection edit --launch-user (-lu) " + f"{bcolors.ENDC}") + return + logging.debug(f"No linked launch credential found for record {record_uid}; " + f"ephemeral JIT is enabled, proceeding without a static " + f"launch credential") + else: + # KeeperRDP / KeeperSSH Proxy: validate the record flag set via + # record-update -r pamSettings=$JSON:{"portForward":{"allowKeeperRDPProxy":true}} + # record-update -r pamSettings=$JSON:{"portForward":{"allowKeeperSSHProxy":true}} + # Unlike KeeperDB-Proxy's writer/reader bucket mismatch, RDP + # and SSH only ever store this flag in `portForward` — no + # `connection` fallback. + proxy_flag_key = 'allowKeeperRDPProxy' if _cls_is_rdp else 'allowKeeperSSHProxy' + proxy_label = 'KeeperRDP Proxy' if _cls_is_rdp else 'KeeperSSH Proxy' + allow_proxy = is_keeperrdp_proxy if _cls_is_rdp else is_keeperssh_proxy + if not allow_proxy: + print(f"{bcolors.FAIL}{proxy_label} is not enabled for record {record_uid}.{bcolors.ENDC}") + print(f"{bcolors.WARNING}Enable it on the record's pamSettings.portForward bucket: " + f"set {bcolors.OKBLUE}{proxy_flag_key}=true{bcolors.WARNING} (e.g. via " + f"{bcolors.OKBLUE}record-update -r {record_uid} --force " + f"pamSettings=$JSON:{{\"portForward\":{{\"{proxy_flag_key}\":true}}}}" + f"{bcolors.ENDC})") + return + # Launch-credential pre-flight (same gate KeeperDB uses). + # The gateway resolves the linked pamUser's login/password + # into tunnel_params via _process_user_record, so failing + # without one yields the cryptic + # `gateway_webrtcaction_missing_rdp_credentials`. + # --credential supplies its own credential and bypasses this + # requirement entirely — independent of the ephemeral fallback + # above (see plan Section 2.1 NB2). + if not launch_credential: + _est, _ett, _tk = get_keeper_tokens(params) + _existing_cfg = get_config_uid(params, _est, _ett, record_uid) + _proxy_dag = TunnelDAG(params, _est, _ett, _existing_cfg, transmission_key=_tk) + if not _proxy_dag.check_if_resource_has_launch_credential(record_uid): + if not _resource_has_ephemeral_jit(params, record_uid): + print(f"{bcolors.FAIL}No Launch Credentials assigned to record \"{record_uid}\". " + f"Please assign launch credentials before using --proxy.{bcolors.ENDC}") + print(f"{bcolors.WARNING}Use: " + f"{bcolors.OKBLUE}pam connection edit --launch-user (-lu) " + f"{bcolors.ENDC}") + return + logging.debug(f"No linked launch credential found for record {record_uid}; " + f"ephemeral JIT is enabled, proceeding without a static " + f"launch credential") + + if launch_credential: + if not _pam_allow_supply_user(pam_settings_value): + print(f"{bcolors.FAIL}--credential requires the \"Allow users to select " + f"credentials from their vault\" option to be enabled on " + f"record \"{record_uid}\".{bcolors.ENDC}") + print(f"{bcolors.WARNING}Enable it with: {bcolors.OKBLUE}record-update " + f"-r {record_uid} pamSettings=$JSON:" + f'{{"connection":{{"allowSupplyUser":true}}}}{bcolors.ENDC}') + return + credential_record_uid = _resolve_credential_record(params, launch_credential) + if not credential_record_uid: + print(f"{bcolors.FAIL}--credential record \"{launch_credential}\" not found.{bcolors.ENDC}") + return + credential_fields = _extract_user_record_credentials(params, credential_record_uid) + credential_type_override = 'userSupplied' + # Shared dict shape for both RDP and SSH: RDP's Gateway-side + # get_rdp_proxy_parameters never reads private_key/passphrase, + # so carrying the two extra keys through for RDP is harmless. + credential_data_override = { + 'username': credential_fields.get('username', ''), + 'password': credential_fields.get('password', ''), + 'private_key': credential_fields.get('private_key') or '', + 'passphrase': credential_fields.get('passphrase') or '', + } + logging.debug(f"Using --credential override for tunnel start on record " + f"{record_uid}: credential record={credential_record_uid}") # Get target host and port if allow_supply_host: @@ -1074,18 +1384,118 @@ def execute(self, params, **kwargs): f"WebRTC cleanup is best-effort.{bcolors.ENDC}") return - result = start_rust_tunnel(params, record_uid, gateway_uid, host, port, seed, target_host, target_port, socks, trickle_ice, record.title, allow_supply_host=allow_supply_host, two_factor_value=two_factor_value) + # Ephemeral-JIT account creation on the gateway takes 30-90s (mirrors + # the connection path's timeout bump in terminal_connection.py) — only + # relevant for proxy tunnels, and only when no explicit --credential + # override is in play (an override skips ephemeral entirely). + is_ephemeral = ( + (is_keeperdb_proxy or is_keeperrdp_proxy or is_keeperssh_proxy) + and not credential_type_override + and _resource_has_ephemeral_jit(params, record_uid) + ) + + # Which proxy-ready banner (if any) to show once the tunnel actually + # connects — see TunnelSignalHandler's 'connected' callback in + # tunnel_helpers.py, which is where it's actually printed. + proxy_kind = 'db' if is_keeperdb_proxy else 'rdp' if is_keeperrdp_proxy else 'ssh' if is_keeperssh_proxy else None + + # When `allow_ephemeral_fallback` is on we're about to attempt + # the legacy default port and the pre-probe said it was free. + # In the rare TOCTOU window the bind can still race and lose, + # in which case Rust (via pyo3-log) and Commander both emit + # noisy ERROR-level lines ("Failed to bind … os error 10048", + # the pyo3 PyBind wrapper, and Commander's "Error in + # start_rust_tunnel"). Suppress ERROR-level logging during + # this first attempt so the user only sees those if the retry + # also fails. Reset before the retry so any second-attempt + # errors surface normally. + if allow_ephemeral_fallback: + logging.disable(logging.ERROR) + try: + result = start_rust_tunnel( + params, record_uid, gateway_uid, host, port, seed, + target_host, target_port, socks, trickle_ice, + record.title, allow_supply_host=allow_supply_host, + two_factor_value=two_factor_value, + credential_type=credential_type_override, + credential_data=credential_data_override, + is_ephemeral=is_ephemeral, + proxy_kind=proxy_kind, + proxy_db_type=db_type_for_banner, + ) + finally: + logging.disable(logging.NOTSET) + else: + result = start_rust_tunnel( + params, record_uid, gateway_uid, host, port, seed, + target_host, target_port, socks, trickle_ice, + record.title, allow_supply_host=allow_supply_host, + two_factor_value=two_factor_value, + credential_type=credential_type_override, + credential_data=credential_data_override, + is_ephemeral=is_ephemeral, + proxy_kind=proxy_kind, + proxy_db_type=db_type_for_banner, + ) + + # No-`--port` legacy default fallback. If Rust failed to bind the + # preferred LEGACY_DEFAULT_TUNNEL_PORT because something else + # already owns it (concurrent `pam tunnel start`, an unrelated + # service, etc.), retry with port=0 so the OS picks an + # ephemeral port. We let the actual bind decide instead of + # trusting the (now stale) pre-probe. Only triggers when the + # user did not pass `--port`. + if ( + allow_ephemeral_fallback + and (not result or not result.get("success")) + ): + err_text = ((result or {}).get("error") or "").lower() + # Windows reports SO_EXCLUSIVEADDRUSE collisions as + # WSAEACCES (os error 10013, "An attempt was made to access + # a socket in a way forbidden by its access permissions") + # rather than the more familiar WSAEADDRINUSE (10048). + # Unix uses EADDRINUSE / "Address already in use". + looks_like_bind_taken = ( + "address already in use" in err_text + or "wsaeaddrinuse" in err_text + or "wsaeaccess" in err_text + or "access permissions" in err_text + or ("bind" in err_text and "10013" in err_text) + or ("bind" in err_text and "10048" in err_text) + or ("bind" in err_text and "in use" in err_text) + or ("bind" in err_text and "forbidden" in err_text) + ) + if looks_like_bind_taken: + logging.debug( + f"start_rust_tunnel: legacy default port {port} unavailable " + f"({err_text!r}); retrying with ephemeral (port=0)" + ) + port = 0 + result = start_rust_tunnel( + params, record_uid, gateway_uid, host, port, seed, + target_host, target_port, socks, trickle_ice, + record.title, allow_supply_host=allow_supply_host, + two_factor_value=two_factor_value, + credential_type=credential_type_override, + credential_data=credential_data_override, + is_ephemeral=is_ephemeral, + proxy_kind=proxy_kind, + proxy_db_type=db_type_for_banner, + ) if result and result.get("success"): - # When --proxy was used, print the KeeperDB Proxy info banner once. - # Local listener is already bound (start_rust_tunnel returns the - # actual_local_listen_addr from Rust), so the connect string is - # valid even though the WebRTC handshake may still be in progress. - # Single call covers interactive, foreground, run, and background- - # child modes — the background parent returns earlier and never - # reaches this branch. - if is_keeperdb_proxy: - self._print_keeperdb_proxy_banner(host, port, db_type_for_banner) + # The proxy-ready banner (KeeperDB/KeeperRDP/KeeperSSH) is *not* + # printed here. `result["success"]` only means the WebRTC offer + # was accepted, not that the tunnel actually connected — for a + # proxy tunnel, the gateway still has to spawn the proxy and + # authenticate against the real target before a valid SDP answer + # is even generated, which can fail (bad credentials, upstream + # timeout) well after this point. Printing the banner here would + # show "ready" for a tunnel that then immediately fails. Instead, + # `TunnelSignalHandler`'s WebRTC 'connected' callback + # (tunnel_helpers.py) prints it once the tunnel is actually up, + # right alongside "Connection established successfully" — which + # also never fires at all if the proxy setup failed. # Workflow lease expiry handling. # # At expiresOn we close the tube (stops new channels, sends @@ -1497,48 +1907,60 @@ def _parse_stun(cls, data: bytes) -> dict: # ── individual Python-side tests ────────────────────────────────────────── @classmethod - def _test_https(cls, hostname: str, port: int = 443) -> Tuple[bool, str, int]: + def _test_https(cls, hostname: str, port: int = 443, proxies=None, verify=True) -> Tuple[bool, str, int]: """Returns (passed, detail, ms).""" t0 = time.monotonic() - conn = None + resp = None try: - ctx = ssl.create_default_context() - conn = http.client.HTTPSConnection(hostname, port=port, context=ctx, timeout=10) - conn.request('GET', '/', headers={'User-Agent': 'keeper-pam-diagnose/1.0'}) - resp = conn.getresponse() + resp = requests.get( + f'https://{hostname}:{port}/', + headers={'User-Agent': 'keeper-pam-diagnose/1.0'}, + proxies=proxies, + verify=verify, + timeout=10, + stream=True, + ) ms = int((time.monotonic() - t0) * 1000) - return 100 <= resp.status < 400, f'HTTP {resp.status} (reachable)', ms + return 100 <= resp.status_code < 400, f'HTTP {resp.status_code} (reachable)', ms except Exception as exc: return False, str(exc)[:60], int((time.monotonic() - t0) * 1000) finally: - if conn: - try: conn.close() - except Exception: pass + if resp is not None: + try: + resp.close() + except Exception: + pass @classmethod - def _test_websocket(cls, hostname: str, port: int = 443) -> Tuple[bool, str, int]: + def _test_websocket(cls, hostname: str, port: int = 443, proxies=None, verify=True) -> Tuple[bool, str, int]: """HTTP Upgrade probe — any 4xx means the server is reachable.""" t0 = time.monotonic() - conn = None + resp = None try: - ctx = ssl.create_default_context() - conn = http.client.HTTPSConnection(hostname, port=port, context=ctx, timeout=10) - conn.request('GET', '/', headers={ - 'Upgrade': 'websocket', - 'Connection': 'Upgrade', - 'Sec-WebSocket-Key': 'dGhlIHNhbXBsZSBub25jZQ==', - 'Sec-WebSocket-Version': '13', - 'User-Agent': 'keeper-pam-diagnose/1.0', - }) - resp = conn.getresponse() + resp = requests.get( + f'https://{hostname}:{port}/', + headers={ + 'Upgrade': 'websocket', + 'Connection': 'Upgrade', + 'Sec-WebSocket-Key': 'dGhlIHNhbXBsZSBub25jZQ==', + 'Sec-WebSocket-Version': '13', + 'User-Agent': 'keeper-pam-diagnose/1.0', + }, + proxies=proxies, + verify=verify, + timeout=10, + stream=True, + ) ms = int((time.monotonic() - t0) * 1000) - return 100 <= resp.status < 400, f'HTTP {resp.status}', ms + return 100 <= resp.status_code < 400, f'HTTP {resp.status_code}', ms except Exception as exc: return False, str(exc)[:60], int((time.monotonic() - t0) * 1000) finally: - if conn: - try: conn.close() - except Exception: pass + if resp is not None: + try: + resp.close() + except Exception: + pass @classmethod def _test_tcp_stun(cls, hostname: str) -> Tuple[bool, str, int, Optional[str]]: @@ -1728,10 +2150,12 @@ def _record(name: str, passed: bool, detail: str, ms: int): except Exception as exc: _record(f'DNS {server_host}', False, str(exc)[:60], int((time.monotonic() - t0) * 1000)) - passed, detail, ms = self._test_https(server_host) + passed, detail, ms = self._test_https( + server_host, proxies=params.rest_context.proxies, verify=params.ssl_verify) _record(f'HTTPS {server_host}:443', passed, detail, ms) - passed, detail, ms = self._test_websocket(connect_host) + passed, detail, ms = self._test_websocket( + connect_host, proxies=params.rest_context.proxies, verify=params.ssl_verify) _record(f'WebSocket {connect_host}:443', passed, detail, ms) print() diff --git a/keepercommander/commands/universalsecretsync.py b/keepercommander/commands/universalsecretsync.py index 5bbda109e..2cdde30f5 100644 --- a/keepercommander/commands/universalsecretsync.py +++ b/keepercommander/commands/universalsecretsync.py @@ -79,7 +79,7 @@ def print_uss_configurations_list(params, format_type='table'): # Only process these specific configuration types uss_supported_types = ('pamGcpConfiguration', 'pamAzureConfiguration', 'pamAwsConfiguration', - 'pamGitHubConfiguration') + 'pamGitHubConfiguration', 'pamHashiCorpConfiguration') configs_data = [] for record in configurations: @@ -115,6 +115,21 @@ def print_uss_configurations_list(params, format_type='table'): # GitHub-specific fields are nested under the 'github' key. github_data = config_data.get('github') or {} + # HashiCorp-specific fields are nested under the 'hashicorp' key. + hashicorp_data = config_data.get('hashicorp') or {} + + # Decrypt HashiCorp vault_base_url if present + vault_base_url = 'N/A' + vault_base_url_encrypted = hashicorp_data.get('vaultBaseUrl') + if vault_base_url_encrypted: + try: + vault_base_url_bytes = crypto.decrypt_aes_v2( + utils.base64_url_decode(vault_base_url_encrypted), record.record_key) + vault_base_url = vault_base_url_bytes.decode('utf-8') + except Exception as e: + logging.debug(f"Failed to decrypt vault_base_url for record {record.record_uid}: {e}") + vault_base_url = 'N/A' + # Decrypt vault_name if present. The router stores it under the # 'vaultName' key as a base64-url string of the encrypted bytes. vault_name = 'N/A' @@ -206,6 +221,7 @@ def print_uss_configurations_list(params, format_type='table'): 'owner': owner, 'organization_visibility': org_visibility_str, 'repos': repo_names, + 'vault_base_url': vault_base_url, }) except Exception as e: # Skip records that fail to load or don't have USS config @@ -227,7 +243,7 @@ def print_uss_configurations_list(params, format_type='table'): # Display as simple summary table table = [] headers = ['Network UID', 'Title', 'Type', 'Enabled', 'Dry Run', 'Folders', 'Vault Name', 'Sync Identity', - 'Scope', 'Owner', 'Org Visibility', 'Repos'] + 'Scope', 'Owner', 'Org Visibility', 'Repos', 'Vault Base URL'] for config in configs_data: enabled_str = f"{bcolors.OKGREEN}Yes{bcolors.ENDC}" if config['enabled'] else f"{bcolors.FAIL}No{bcolors.ENDC}" @@ -249,7 +265,8 @@ def print_uss_configurations_list(params, format_type='table'): config.get('scope', 'N/A'), config.get('owner', 'N/A'), config.get('organization_visibility', 'N/A'), - repos_str + repos_str, + config.get('vault_base_url', 'N/A') ] table.append(row) @@ -273,7 +290,7 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): # Check if it's a supported USS configuration type uss_supported_types = ('pamGcpConfiguration', 'pamAzureConfiguration', 'pamAwsConfiguration', - 'pamGitHubConfiguration') + 'pamGitHubConfiguration', 'pamHashiCorpConfiguration') if not isinstance(network, vault.TypedRecord) or network.record_type not in uss_supported_types: if format_type == 'json': return json.dumps({"error": f'Record "{network_uid}" is not a USS configuration'}) @@ -310,6 +327,21 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): # GitHub-specific fields are nested under the 'github' key. github_data = config_data.get('github') or {} + # HashiCorp-specific fields are nested under the 'hashicorp' key. + hashicorp_data = config_data.get('hashicorp') or {} + + # Decrypt HashiCorp vault_base_url if present + vault_base_url = 'N/A' + vault_base_url_encrypted = hashicorp_data.get('vaultBaseUrl') + if vault_base_url_encrypted: + try: + vault_base_url_bytes = crypto.decrypt_aes_v2( + utils.base64_url_decode(vault_base_url_encrypted), network.record_key) + vault_base_url = vault_base_url_bytes.decode('utf-8') + except Exception as e: + logging.debug(f"Failed to decrypt vault_base_url for network {network.record_uid}: {e}") + vault_base_url = 'N/A' + # Decrypt vault_name if present. The router stores it under the # 'vaultName' key as a base64-url string of the encrypted bytes. vault_name = 'N/A' @@ -438,6 +470,7 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): 'owner': owner, 'organization_visibility': org_visibility_str, 'repos': repo_names, + 'vault_base_url': vault_base_url, 'folders': [] } @@ -473,6 +506,7 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): table.append(['Owner', owner]) table.append(['Org Visibility', org_visibility_str]) table.append(['Repos', ', '.join(repo_names) if repo_names else 'None']) + table.append(['Vault Base URL', vault_base_url]) table.append(['', '']) # Blank row separator # Display folder sync details @@ -521,6 +555,7 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): class PAMUniversalSyncConfigAddCommand(Command): parser = argparse.ArgumentParser(prog='pam universal-sync-config add') + parser.add_argument('--network', '-n', required=True, dest='network', action='store', help='Network UID or name to configure universal sync') parser.add_argument('--enabled', '-e', dest='enabled', action='store', @@ -543,6 +578,14 @@ class PAMUniversalSyncConfigAddCommand(Command): help='Repository visibility to sync when scope is organization') parser.add_argument('--repo', '-r', dest='repo', action='append', help='GitHub repository name to sync (can be specified multiple times; scope must be repository)') + parser.add_argument('--vault-base-url', '-vbu', dest='vault_base_url', action='store', + help='HashiCorp Vault Base URL (e.g., https://vault.company.com:8200)') + parser.add_argument('--vault-token', '-vt', dest='vault_token', action='store', + help='HashiCorp Vault Token (optional; syncIdentity takes precedence)') + parser.add_argument('--vault-namespace', '-vns', dest='vault_namespace', action='store', + help='HashiCorp Vault Namespace (optional; leave blank for Community Edition)') + parser.add_argument('--vault-mount-path', '-vmp', dest='vault_mount_path', action='store', + help='HashiCorp Vault KV Mount Path (optional; defaults to "secret")') def get_parser(self): return PAMUniversalSyncConfigAddCommand.parser @@ -616,6 +659,30 @@ def execute(self, params, **kwargs): repo_obj.name = crypto.encrypt_aes_v2(repo_bytes, network.record_key) rq.github.repos.append(repo_obj) + vault_base_url = kwargs.get('vault_base_url') + if vault_base_url: + vault_base_url_bytes = string_to_bytes(vault_base_url) + encrypted_vault_base_url = crypto.encrypt_aes_v2(vault_base_url_bytes, network.record_key) + rq.hashicorp.vaultBaseUrl = encrypted_vault_base_url + + vault_token = kwargs.get('vault_token') + if vault_token: + vault_token_bytes = string_to_bytes(vault_token) + encrypted_vault_token = crypto.encrypt_aes_v2(vault_token_bytes, network.record_key) + rq.hashicorp.vaultToken = encrypted_vault_token + + vault_namespace = kwargs.get('vault_namespace') + if vault_namespace: + vault_namespace_bytes = string_to_bytes(vault_namespace) + encrypted_vault_namespace = crypto.encrypt_aes_v2(vault_namespace_bytes, network.record_key) + rq.hashicorp.vaultNamespace = encrypted_vault_namespace + + vault_mount_path = kwargs.get('vault_mount_path') + if vault_mount_path: + vault_mount_path_bytes = string_to_bytes(vault_mount_path) + encrypted_vault_mount_path = crypto.encrypt_aes_v2(vault_mount_path_bytes, network.record_key) + rq.hashicorp.vaultMountPath = encrypted_vault_mount_path + encrypted_session_token, encrypted_transmission_key, transmission_key = get_keeper_tokens(params) try: @@ -651,6 +718,14 @@ class PAMUniversalSyncConfigEditCommand(Command): help='Repository visibility to sync when scope is organization') parser.add_argument('--repo', '-r', dest='repo', action='append', help='GitHub repository name to sync (can be specified multiple times; scope must be repository)') + parser.add_argument('--vault-base-url', '-vbu', dest='vault_base_url', action='store', + help='HashiCorp Vault Base URL (e.g., https://vault.company.com:8200)') + parser.add_argument('--vault-token', '-vt', dest='vault_token', action='store', + help='HashiCorp Vault Token (optional; syncIdentity takes precedence)') + parser.add_argument('--vault-namespace', '-vns', dest='vault_namespace', action='store', + help='HashiCorp Vault Namespace (optional; leave blank for Community Edition)') + parser.add_argument('--vault-mount-path', '-vmp', dest='vault_mount_path', action='store', + help='HashiCorp Vault KV Mount Path (optional; defaults to "secret")') def get_parser(self): return PAMUniversalSyncConfigEditCommand.parser @@ -786,6 +861,41 @@ def execute(self, params, **kwargs): repo_obj.name = utils.base64_url_decode(existing_repo) rq.github.repos.append(repo_obj) + # HashiCorp-specific fields live under the nested 'hashicorp' object + existing_hashicorp = existing_config.get('hashicorp') or {} + + vault_base_url = kwargs.get('vault_base_url') + if vault_base_url: + vault_base_url_bytes = string_to_bytes(vault_base_url) + encrypted_vault_base_url = crypto.encrypt_aes_v2(vault_base_url_bytes, network.record_key) + rq.hashicorp.vaultBaseUrl = encrypted_vault_base_url + elif existing_hashicorp.get('vaultBaseUrl'): + rq.hashicorp.vaultBaseUrl = utils.base64_url_decode(existing_hashicorp['vaultBaseUrl']) + + vault_token = kwargs.get('vault_token') + if vault_token: + vault_token_bytes = string_to_bytes(vault_token) + encrypted_vault_token = crypto.encrypt_aes_v2(vault_token_bytes, network.record_key) + rq.hashicorp.vaultToken = encrypted_vault_token + elif existing_hashicorp.get('vaultToken'): + rq.hashicorp.vaultToken = utils.base64_url_decode(existing_hashicorp['vaultToken']) + + vault_namespace = kwargs.get('vault_namespace') + if vault_namespace: + vault_namespace_bytes = string_to_bytes(vault_namespace) + encrypted_vault_namespace = crypto.encrypt_aes_v2(vault_namespace_bytes, network.record_key) + rq.hashicorp.vaultNamespace = encrypted_vault_namespace + elif existing_hashicorp.get('vaultNamespace'): + rq.hashicorp.vaultNamespace = utils.base64_url_decode(existing_hashicorp['vaultNamespace']) + + vault_mount_path = kwargs.get('vault_mount_path') + if vault_mount_path: + vault_mount_path_bytes = string_to_bytes(vault_mount_path) + encrypted_vault_mount_path = crypto.encrypt_aes_v2(vault_mount_path_bytes, network.record_key) + rq.hashicorp.vaultMountPath = encrypted_vault_mount_path + elif existing_hashicorp.get('vaultMountPath'): + rq.hashicorp.vaultMountPath = utils.base64_url_decode(existing_hashicorp['vaultMountPath']) + encrypted_session_token, encrypted_transmission_key, transmission_key = get_keeper_tokens(params) try: diff --git a/keepercommander/commands/utils.py b/keepercommander/commands/utils.py index 0cfb4a0e4..3be980c95 100644 --- a/keepercommander/commands/utils.py +++ b/keepercommander/commands/utils.py @@ -45,7 +45,7 @@ from ..error import CommandError from ..generator import ( KeeperPasswordGenerator, DicewarePasswordGenerator, CryptoPassphraseGenerator, - KeeperPassphraseGenerator, PASSPHRASE_SEPARATOR_HELP, + KeeperPassphraseGenerator, AESKeyGenerator, PASSPHRASE_SEPARATOR_HELP, _parse_passphrase_separator_token ) from ..params import KeeperParams, LAST_RECORD_UID, LAST_FOLDER_UID, LAST_SHARED_FOLDER_UID from ..proto import ssocloud_pb2, enterprise_pb2, APIRequest_pb2 @@ -81,7 +81,6 @@ def add(self, other): self.failed += other.failed self.vault_changed = self.vault_changed or other.vault_changed - class DeletionValidator: """Centralized validation for deletion operations.""" @@ -441,7 +440,10 @@ def register_command_info(aliases, command_info): help='Optional. File path to word list') crypto_group = generate_parser.add_argument_group('Crypto') -crypto_group.add_argument('--crypto', dest='crypto', action='store_true', help='Generate crypto wallet passphrase') +crypto_group.add_argument('--crypto', dest='crypto', action='store_true', help='Generate crypto wallet private key') + +aes_key_group = generate_parser.add_argument_group('AES 256-bit Key') +aes_key_group.add_argument('--aes-key', dest='aes_key', action='store_true', help='AES 256 bit encryption key. Base64 URL safe encoded') reset_password_parser = argparse.ArgumentParser(prog='reset-password', description='Reset Master Password') reset_password_parser.add_argument('--delete-sso', dest='delete_alternate', action='store_true', @@ -521,6 +523,8 @@ def execute(self, params, **kwargs): class ThisDeviceCommand(Command): + skip_sync_on_auth = True + def get_parser(self): return this_device_parser @@ -2237,13 +2241,15 @@ def execute(self, params, number=None, no_breachwatch=None, if kwargs.get('crypto') is True: kpg = CryptoPassphraseGenerator() + elif kwargs.get('aes_key') is True: + kpg = AESKeyGenerator() elif kwargs.get('passphrase') is True: from ..enforcement import PasswordComplexityEnforcer policy = PasswordComplexityEnforcer.get_policy(params) word_count = length if length != 20 else None pp_separator = kwargs.get('pp_separator') if isinstance(pp_separator, str) and pp_separator.strip(): - pp_separator, pp_sep_error = generator._parse_passphrase_separator_token( + pp_separator, pp_sep_error = _parse_passphrase_separator_token( pp_separator.strip()) if pp_sep_error: logging.error(pp_sep_error) diff --git a/keepercommander/discovery_common/__version__.py b/keepercommander/discovery_common/__version__.py index 5eba4ad50..f5e08a61e 100644 --- a/keepercommander/discovery_common/__version__.py +++ b/keepercommander/discovery_common/__version__.py @@ -1 +1 @@ -__version__ = '1.1.23' +__version__ = '1.1.25' diff --git a/keepercommander/discovery_common/record_link.py b/keepercommander/discovery_common/record_link.py index 89cb72824..947191a20 100644 --- a/keepercommander/discovery_common/record_link.py +++ b/keepercommander/discovery_common/record_link.py @@ -92,7 +92,8 @@ def dag(self) -> DAG: fail_on_corrupt=self.fail_on_corrupt, log_prefix=self.log_prefix, save_batch_count=self.save_batch_count, - agent=self.agent) + agent=self.agent, + path_aware=True) sync_point = self._dag.load(sync_point=0) self.logger.debug(f"the record linking sync point is {sync_point or 0}") if not self.dag.has_graph: diff --git a/keepercommander/discovery_common/rm_types.py b/keepercommander/discovery_common/rm_types.py index 6829da93a..1a2e62b9e 100644 --- a/keepercommander/discovery_common/rm_types.py +++ b/keepercommander/discovery_common/rm_types.py @@ -107,10 +107,12 @@ class RmMappedUser(BaseModel): :param id: Provider identifier for the user. :param name: Human-readable user name. :param role_ids: Ids of the roles this user holds (references RmMappedRole.id). + :param group_ids: Ids of the groups this user belongs to (references RmMappedGroup.id). """ id: str name: Optional[str] = None role_ids: List[str] = [] + group_ids: List[str] = [] class RmMappedGroup(BaseModel): diff --git a/keepercommander/enterprise.py b/keepercommander/enterprise.py index 74808877f..f7de2b747 100644 --- a/keepercommander/enterprise.py +++ b/keepercommander/enterprise.py @@ -185,6 +185,7 @@ def load(self, params): # type: (KeeperParams) -> None params.enterprise['keys'] = keys entities = set() + root_restrict_visibility = None while True: rq = proto.EnterpriseDataRequest() if self._continuationToken: @@ -202,6 +203,8 @@ def load(self, params): # type: (KeeperParams) -> None params.enterprise['enterprise_name'] = self._enterprise.enterprise_name if rs.generalData.distributor: params.enterprise['distributor'] = True + if rs.HasField('generalData'): + root_restrict_visibility = rs.generalData.restrictVisibility for ed in rs.data: entities.add(ed.entity) @@ -212,6 +215,11 @@ def load(self, params): # type: (KeeperParams) -> None self._continuationToken = rs.continuationToken if not rs.hasMore: break + if root_restrict_visibility is not None: + root_node = next((x for x in params.enterprise.get('nodes', []) if not x.get('parent_id')), None) + if root_node: + _set_or_remove(root_node, 'restrict_visibility', + True if root_restrict_visibility else None) if proto.MANAGED_NODES in entities: try: self.load_missing_role_keys(params) @@ -456,8 +464,9 @@ def to_keeper_entity(self, proto_entity, keeper_entity): # type: (proto.Node, d _set_or_remove(keeper_entity, 'rsa_enabled', True if proto_entity.rsaEnabled else None) _set_or_remove(keeper_entity, 'sso_service_provider_id', proto_entity.ssoServiceProviderId if proto_entity.ssoServiceProviderId > 0 else None) - _set_or_remove(keeper_entity, 'restrict_visibility', - proto_entity.restrictVisibility if proto_entity.restrictVisibility else None) + if keeper_entity.get('parent_id'): + _set_or_remove(keeper_entity, 'restrict_visibility', + proto_entity.restrictVisibility if proto_entity.restrictVisibility else None) data = {} if 'encrypted_data' in keeper_entity: diff --git a/keepercommander/generator.py b/keepercommander/generator.py index 14955376e..ad744b177 100644 --- a/keepercommander/generator.py +++ b/keepercommander/generator.py @@ -19,7 +19,7 @@ from typing import Optional, List, Iterator, Sequence, Tuple from collections import namedtuple -from . import crypto +from . import crypto, utils from Cryptodome.Random.random import shuffle @@ -478,6 +478,11 @@ def create_from_policy(cls, policy, length_override=None, separator_override=Non ) +class AESKeyGenerator(PasswordGenerator): + def generate(self): + return utils.base64_url_encode(utils.generate_aes_key()) + + class CryptoPassphraseGenerator(PasswordGenerator): def __init__(self): self._vocabulary = None # type: Optional[List[str]] diff --git a/keepercommander/importer/commands.py b/keepercommander/importer/commands.py index 223f5525b..83f2d75ed 100644 --- a/keepercommander/importer/commands.py +++ b/keepercommander/importer/commands.py @@ -155,6 +155,7 @@ def _cyberark_skip_arg(value): apply_membership_parser = argparse.ArgumentParser(prog='apply-membership', description='Loads shared folder membership from JSON file') apply_membership_parser.add_argument('--full-sync', dest='full_sync', action='store_true', help='Update and remove membership also.') +apply_membership_parser.add_argument('--unsafe', dest='unsafe', action='store_true', help='Combined with --full-sync. Will remove yourself from folders where you have no permission.') apply_membership_parser.add_argument('name', type=str, nargs='?', help='Input file name. "shared_folder_membership.json" if omitted.') apply_membership_parser.error = raise_parse_exception apply_membership_parser.exit = suppress_exit @@ -547,8 +548,9 @@ def execute(self, params, **kwargs): teams.append(obj) full_sync = kwargs.get('full_sync') is True + unsafe = kwargs.get('unsafe') is True if len(shared_folders) > 0: - imp_exp.import_user_permissions(params, shared_folders, full_sync) + imp_exp.import_user_permissions(params, shared_folders, full_sync, unsafe) if len(teams) > 0: imp_exp.import_teams(params, teams, full_sync) diff --git a/keepercommander/importer/cyberark/pam/__init__.py b/keepercommander/importer/cyberark/pam/__init__.py index 21bbfdf4e..abb4c951e 100644 --- a/keepercommander/importer/cyberark/pam/__init__.py +++ b/keepercommander/importer/cyberark/pam/__init__.py @@ -20,6 +20,7 @@ RECORD_TYPE_PAM_DATABASE, RECORD_TYPE_PAM_DIRECTORY, RECORD_TYPE_PAM_MACHINE, + RECORD_TYPE_PAM_REMOTE_BROWSER, ROTATION_UNMAPPED, SCHEDULE_ON_DEMAND, SYSTEM_SAFES, @@ -101,6 +102,7 @@ "RECORD_TYPE_PAM_DATABASE", "RECORD_TYPE_PAM_DIRECTORY", "RECORD_TYPE_PAM_MACHINE", + "RECORD_TYPE_PAM_REMOTE_BROWSER", "ROTATION_UNMAPPED", "SCHEDULE_ON_DEMAND", "IDENTITY_LOGIN_SUCCESS", diff --git a/keepercommander/importer/cyberark/pam/account_mapper.py b/keepercommander/importer/cyberark/pam/account_mapper.py index 68de82b8a..7eeadaa0f 100644 --- a/keepercommander/importer/cyberark/pam/account_mapper.py +++ b/keepercommander/importer/cyberark/pam/account_mapper.py @@ -1,4 +1,4 @@ -# _ __ +# _ __ # | |/ /___ ___ _ __ ___ _ _ ® # | ' resolved schedule (or # ``None`` once we've decided the platform inherits the master # policy default). One CyberArk API call per unique platformId. self._platform_schedule_cache: Dict[str, Optional[dict]] = {} - # Diagnostic counter — how many accounts received a platform-level + # Diagnostic counter - how many accounts received a platform-level # schedule vs. inherited the master default. Surfaced in the # cyberark_import.py orchestrator output so operators can verify # the override was applied. self.platform_schedule_overrides: Dict[str, int] = {} - # Per-platform session-recording overrides. Maps platformId → + # Per-platform session-recording overrides. Maps platformId -> # ("on"|"off"|None) for graphical session recording. ``None`` - # means we couldn't determine the policy — caller falls back to + # means we couldn't determine the policy - caller falls back to # whatever the master policy / Keeper default dictates. Only one # API call per unique platformId thanks to caching. self._platform_session_cache: Dict[str, Optional[Tuple[str, str]]] = {} @@ -141,7 +145,7 @@ def __init__(self, platform_map_override: Optional[dict] = None, # on every Keeper PAM resource record from the same platform so # the source-system context is preserved post-migration. self._platform_metadata_cache: Dict[str, List[dict]] = {} - # Per-platform CyberArk passwordGenRules → Keeper + # Per-platform CyberArk passwordGenRules -> Keeper # ``rotation_settings.password_complexity`` translation cache. # Populated lazily on first hit of each platformId. self._platform_complexity_cache: Dict[str, Optional[str]] = {} @@ -161,6 +165,164 @@ def __init__(self, platform_map_override: Optional[dict] = None, "PerformPeriodicChange", "rotatePasswordsAutomatically", ) + # These values are already represented by canonical login fields. + _LOGIN_METADATA_TOP_LEVEL_EXCLUSIONS = frozenset({ + "id", "name", "platformid", "address", "username", "safename", + "platformaccountproperties", + }) + _LOGIN_METADATA_PROPERTY_EXCLUSIONS = frozenset({ + "url", "itemname", "logondomain", + "consoleurl", "portalurl", "loginurl", + }) + # Retrieved credentials belong only in the Keeper password field. These + # exact key names guard against an API variant embedding secret material + # in the account payload while allowing safe metadata such as secretType + # and secretManagement to be retained. + _LOGIN_METADATA_SECRET_KEYS = frozenset({ + "password", "secret", "content", "credential", "credentials", + "privatekey", "privatepemkey", "sshkey", + }) + _CLOUD_RBI_DEFAULTS = { + "aws": { + "url": "https://signin.aws.amazon.com/console", + "page": "*.signin.aws.amazon.com", + "username": "#username", + "password": "#password", + "submit": "#signin_button", + }, + "azure": { + "url": "https://portal.azure.com/", + "page": "*.login.microsoftonline.com", + "username": "input[type=email], input[name=loginfmt]", + "password": "input[type=password], input[name=passwd]", + "submit": "input[type=submit], button[type=submit]", + }, + "gcp": { + "url": "https://console.cloud.google.com/", + "page": "accounts.google.com", + "username": "input[type=email], input[name=identifier]", + "password": "input[type=password], input[name=password]", + "submit": "button[type=submit]", + }, + } + + @classmethod + def _cloud_rbi_defaults(cls, cloud: str) -> dict: + return dict(cls._CLOUD_RBI_DEFAULTS.get((cloud or "").lower(), {})) + + @classmethod + def _build_cloud_autofill_targets(cls, cloud: str, url: str) -> str: + defaults = cls._cloud_rbi_defaults(cloud) + page = defaults.get("page") or url or "*" + step = {"page": page} + if defaults.get("username"): + step["username-field"] = defaults["username"] + if defaults.get("password"): + step["password-field"] = defaults["password"] + if defaults.get("submit"): + step["submit"] = defaults["submit"] + return json.dumps([step]) + + @staticmethod + def _first_prop(props: dict, *names: str) -> str: + if not isinstance(props, dict): + return "" + lowered = {str(k).casefold(): k for k in props.keys()} + for name in names: + key = lowered.get(str(name).casefold()) + if key and props.get(key) not in (None, ""): + return str(props[key]).strip() + return "" + + @classmethod + def _metadata_label_to_text(cls, label: str) -> str: + parts = [] + for part in str(label).replace("_", " ").split("."): + part = re.sub(r"(?<=[A-Z])(?=[A-Z][a-z])", " ", part) + part = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", part) + parts.append(" ".join(part.split())) + return ".".join(parts) + + @staticmethod + def _metadata_value_to_text(value) -> str: + if isinstance(value, str): + return value + return json.dumps( + value, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), + ) + + @classmethod + def _build_account_metadata_custom_fields(cls, account: dict) -> List[dict]: + """Flatten CyberArk metadata in the classic importer custom-field style.""" + out: List[dict] = [] + if not isinstance(account, dict): + return out + + properties = account.get("platformAccountProperties") + properties = dict(properties) if isinstance(properties, dict) else {} + for key in ("platformName", "platformId"): + value = account.get(key) + if value not in (None, ""): + properties.setdefault("Platform Name", value) + break + for key in ("deviceType", "device type"): + value = account.get(key) + if value not in (None, ""): + properties.setdefault("Device Type", value) + break + + existing_labels = set() + + def _add(path: str, value) -> None: + if len(out) >= MAX_ACCOUNT_METADATA_FIELDS or value in (None, "", [], {}): + return + if isinstance(value, dict): + for child_key, child_value in value.items(): + child_name = str(child_key) + if child_name.casefold() in cls._LOGIN_METADATA_SECRET_KEYS: + continue + _add(f"{path}.{child_name}", child_value) + return + if isinstance(value, (list, tuple)): + text = cls._metadata_value_to_text(list(value)) + elif isinstance(value, bool): + text = "true" if value else "false" + else: + text = cls._metadata_value_to_text(value) + text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text) + if len(text) > MAX_ACCOUNT_METADATA_VALUE_LEN: + text = text[:MAX_ACCOUNT_METADATA_VALUE_LEN] + label = cls._metadata_label_to_text(path) + label_key = label.casefold() + if not label or label_key in existing_labels: + return + out.append({ + "type": "text", + "label": label[:120], + "value": [text], + }) + existing_labels.add(label_key) + + for key, value in properties.items(): + key_name = str(key) + folded = key_name.casefold() + if (folded in cls._LOGIN_METADATA_PROPERTY_EXCLUSIONS + or folded in cls._LOGIN_METADATA_SECRET_KEYS): + continue + _add(key_name, value) + return out + + @classmethod + def _build_unmapped_login_custom_fields(cls, account: dict) -> List[dict]: + return cls._build_account_metadata_custom_fields(account) + + def _add_source_metadata_to_user(self, user_record: dict, account: dict, platform_id: str) -> None: + """Copy CyberArk source metadata onto the generated pamUser record.""" + account_metadata_fields = self._build_account_metadata_custom_fields(account) + if account_metadata_fields: + user_record.setdefault("custom", []).extend(account_metadata_fields) + @staticmethod def _flatten_rotation_policy(data: dict) -> Dict[str, Any]: """Flatten the platform rotation-policy payload to a single dict. @@ -212,10 +374,10 @@ def _flatten_rotation_policy(data: dict) -> Dict[str, Any]: @staticmethod def _schedule_from_change_block(change: dict) -> Optional[dict]: - """Translate ISPSS ``change`` group → Keeper ``schedule`` dict. + """Translate ISPSS ``change`` group -> Keeper ``schedule`` dict. Mirrors how the Master Policy itself is converted - (``MasterPolicyMapper.days_to_cron``) — the interval value alone + (``MasterPolicyMapper.days_to_cron``) - the interval value alone drives the cadence. CyberArk's ``allowedPeriodic`` / ``performPeriodicChange`` bookkeeping flag is informational (it reflects whether *CyberArk's* CPM auto-triggers the change) and is @@ -247,29 +409,29 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: Calls ``/api/platforms/{platformId}/rotation-policy/`` once per platform and translates the response into a Quartz CRON via ``MasterPolicyMapper.days_to_cron``. Returns ``None`` when the - platform has no custom policy — callers fall back to the master + platform has no custom policy - callers fall back to the master policy default schedule. Honors CyberArk's cascade: - 1. **Per-platform with override** — a recognized exception flag + 1. **Per-platform with override** - a recognized exception flag (``overridesMasterPolicy``, ``isException``, etc.) is true, OR no flag is present but the interval differs from the Master - Policy's own interval → use ``change.interval`` verbatim, + Policy's own interval -> use ``change.interval`` verbatim, converted to CRON exactly like the Master Policy default is - (``allowedPeriodic`` does NOT force on-demand — see + (``allowedPeriodic`` does NOT force on-demand - see ``_schedule_from_change_block``). - 2. **Master-policy exception** — platform listed in + 2. **Master-policy exception** - platform listed in ``/api/platforms/master-rotation-policy/exceptions/`` with a custom change interval (common for Win Local Admins and other platforms that inherit master but have a Master Policy - exception) — used when the per-platform flag says "no + exception) - used when the per-platform flag says "no override" but this separate bulk endpoint disagrees. - 3. **Per-platform without override** — + 3. **Per-platform without override** - no exception detected by (1) or (2): inherit the master - policy's own schedule (``None`` → caller applies + policy's own schedule (``None`` -> caller applies ``default_rotation_schedule``). - 4. **Legacy flat shapes** — older PVWA responses with + 4. **Legacy flat shapes** - older PVWA responses with ``rotateEveryXDays`` / ``passwordChangeDays`` at top level. """ if not platform_id or not self._client: @@ -280,7 +442,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: raw = self._client.fetch_platform_rotation_policy(platform_id) if not raw: msg = ( - f"Platform '{platform_id}' rotation-policy not accessible — " + f"Platform '{platform_id}' rotation-policy not accessible - " "inheriting master policy default." ) if self.strict_policies: @@ -290,7 +452,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: self._platform_schedule_cache[platform_id] = None return None - # ── New ISPSS shape ──────────────────────────────────── + # -- New ISPSS shape ------------------------------------ change = raw.get("change") if isinstance(raw.get("change"), dict) else None if change is not None: try: @@ -302,7 +464,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: # platform's own interval differs from the Master Policy's # configured interval, CyberArk is evidently applying a # platform-specific value regardless of what this tenant calls - # the flag — honor the number it actually returned. + # the flag - honor the number it actually returned. interval_mismatch = ( overrides is None and self._master_change_days > 0 @@ -338,7 +500,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: platform_id, schedule, ) return schedule - # No override, no separate master exception on record — + # No override, no separate master exception on record - # this platform's cadence matches the Master Policy default. # Inherit it (the caller applies ``default_rotation_schedule``, # itself a CRON built from the same master interval), rather @@ -351,7 +513,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: self._platform_schedule_cache[platform_id] = None return None - # overrides is None — this tenant's rotation-policy response + # overrides is None - this tenant's rotation-policy response # doesn't carry any recognized exception flag at all. When we # know the Master Policy's own interval, use it as the source # of truth: a matching interval means no exception exists (the @@ -360,13 +522,13 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: if self._master_change_days > 0: logging.info( "Platform '%s' rotation policy: interval matches master " - "(%d days) — inheriting master policy", platform_id, + "(%d days) - inheriting master policy", platform_id, self._master_change_days, ) self._platform_schedule_cache[platform_id] = None return None - # No flag and no master baseline to compare against — fall back + # No flag and no master baseline to compare against - fall back # to treating the platform's own ``change`` block as authoritative # (legacy behavior for tenants where this is the only info we get). schedule = self._schedule_from_change_block(change) @@ -377,7 +539,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: ) return schedule - # No ``change`` block — check master-policy exceptions before legacy. + # No ``change`` block - check master-policy exceptions before legacy. exc = self._master_exception_schedules.get(platform_id) if exc is not None: schedule = copy.deepcopy(exc) @@ -388,7 +550,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: ) return schedule - # ── Legacy flat shapes (PVWA Self-Hosted, Get-Platforms) ─── + # -- Legacy flat shapes (PVWA Self-Hosted, Get-Platforms) --- flat = self._flatten_rotation_policy(raw) days = 0 for key in self._PLATFORM_ROTATE_DAYS_KEYS: @@ -518,7 +680,7 @@ def _add(label: str, val): change = rotation_raw.get("change") if isinstance(rotation_raw.get("change"), dict) else {} verify = rotation_raw.get("verify") if isinstance(rotation_raw.get("verify"), dict) else {} - # Rotation cadence operational settings — preserved on the + # Rotation cadence operational settings - preserved on the # Keeper record so a user can reproduce them in CyberArk. _add("CyberArk Rotation Interval (days)", change.get("interval")) if change.get("allowedPeriodic") is not None: @@ -562,12 +724,12 @@ def _resolve_platform_session_recording( self, platform_id: str) -> Optional[Tuple[str, str]]: """Return ``("on"|"off", "on"|"off")`` for (graphical, text) recording. - Probes the per-platform endpoints in order (secrets-policy → - platform details → legacy .asmx). When the platform exposes a + Probes the per-platform endpoints in order (secrets-policy -> + platform details -> legacy .asmx). When the platform exposes a ``recordAndSaveSessionActivity`` rule we honor it; that single boolean drives both Keeper recording flags because CyberArk does not split graphical vs. text. Returns ``None`` when no endpoint - answered with usable data — the resource then inherits whatever + answered with usable data - the resource then inherits whatever the master policy / Keeper default decides. """ if not platform_id or not self._client: @@ -576,10 +738,10 @@ def _resolve_platform_session_recording( return self._platform_session_cache[platform_id] # Session-recording flags can live in any of three CyberArk - # surfaces depending on tenant type — none of them is a single + # surfaces depending on tenant type - none of them is a single # source of truth, so we query and merge: # - # 1. ``/PasswordVault/API/Platforms/{id}`` — Get-Platform-Details. + # 1. ``/PasswordVault/API/Platforms/{id}`` - Get-Platform-Details. # On older PVWA Self-Hosted this carries the # ``sessionManagement.recordAndSaveSessionActivity`` boolean. # On Privilege Cloud / ISPSS this DOES NOT include the @@ -588,11 +750,11 @@ def _resolve_platform_session_recording( # ``PSMServerId``/``PSMServerName`` from here as a fallback # indicator (PSM-attached platforms record by default). # 2. ``/PasswordVault/services/PoliciesMgt.asmx/ - # GetPolicyRulesSessionMonitoring`` — the PVWA admin-UI grid + # GetPolicyRulesSessionMonitoring`` - the PVWA admin-UI grid # service. On Privilege Cloud this is the *canonical* place # where ``RecordSession`` / ``MonitorSession`` rules are # surfaced (the user explicitly pointed us here). - # 3. PSMServer presence — heuristic fallback when neither (1) + # 3. PSMServer presence - heuristic fallback when neither (1) # nor (2) has the explicit flag: a platform that has a PSM # ConnectionComponent (``PSM-RDP``, ``PSM-SSH``, ...) # records sessions by default in CyberArk. @@ -607,7 +769,7 @@ def _resolve_platform_session_recording( self._platform_session_cache[platform_id] = None return None - # ── Build a flat ``rules`` dict from every envelope we got ──── + # -- Build a flat ``rules`` dict from every envelope we got ---- rules: Dict[str, Any] = {} for raw in (details, asmx): if not isinstance(raw, dict) or not raw: @@ -656,7 +818,7 @@ def _resolve_platform_session_recording( if k in rules and truthy(rules[k]): record_val = True break - # 3) Heuristic: PSMServer reference present → PSM is engaged + # 3) Heuristic: PSMServer reference present -> PSM is engaged # and CyberArk records by default. Used only when neither # (1) nor (2) provided an answer. if record_val is None: @@ -665,7 +827,7 @@ def _resolve_platform_session_recording( v = rules.get(k) if isinstance(v, str) and v.strip(): logging.debug( - "Platform '%s' has PSMServer=%s — assuming " + "Platform '%s' has PSMServer=%s - assuming " "session recording is on (PSM default)", platform_id, v, ) @@ -769,9 +931,9 @@ def _resolve_from_platform_metadata(self, platform_id: str) -> Optional[dict]: """Map a custom platformId via PVWA's PlatformBaseID / SystemType. Order: - 1. PlatformBaseID matches a built-in (e.g. WinDomain) — use that mapping. - 2. SystemType matches our SYSTEM_TYPE_MAP (e.g. Windows → RDP). - 3. None — caller falls through to keyword guessing. + 1. PlatformBaseID matches a built-in (e.g. WinDomain) - use that mapping. + 2. SystemType matches our SYSTEM_TYPE_MAP (e.g. Windows -> RDP). + 3. None - caller falls through to keyword guessing. """ meta = self._platform_index.get(platform_id) if not meta: @@ -791,8 +953,8 @@ def _infer_operating_system(platform_id: str, protocol: Optional[str]) -> Optional[str]: """Derive ``operating_system`` for a pamMachine from platformId/protocol. - Keyword scan on platformId (Win→windows, Unix/Linux→linux), then - protocol fallback (rdp→windows, ssh→linux). Returns ``None`` when + Keyword scan on platformId (Win->windows, Unix/Linux->linux), then + protocol fallback (rdp->windows, ssh->linux). Returns ``None`` when ambiguous so the field is left unset. """ pid = (platform_id or "").lower() @@ -834,9 +996,10 @@ def _enrich_port_from_details(self, platform_id: str, mapping: dict) -> dict: def map_account(self, account: dict, password: Optional[str] = None, safe_name: str = "") -> Optional[dict]: - """Convert a CyberArk account dict → pam_data record dict. + """Convert a CyberArk account dict -> pam_data record dict. - Returns None if the platformId is completely unknown and has no default. + Accounts with no resolvable platform mapping are returned as standalone + login records instead of speculative PAM resources. """ platform_id = account.get("platformId", "") mapping = self.platform_map.get(platform_id) if platform_id else None @@ -844,10 +1007,10 @@ def map_account(self, account: dict, password: Optional[str] = None, if mapping is None: # Resolution order for unknown / customer-renamed platforms: - # 1. PVWA platform metadata — PlatformBaseID → DEFAULT_PLATFORM_MAP, - # or SystemType → _SYSTEM_TYPE_MAP. Authoritative. + # 1. PVWA platform metadata - PlatformBaseID -> DEFAULT_PLATFORM_MAP, + # or SystemType -> _SYSTEM_TYPE_MAP. Authoritative. # 2. Substring keyword match on platformId / name. - # 3. pamMachine/SSH fallback. + # 3. Standalone login fallback (no PAM resource or nested user). label = platform_id if platform_id else "(empty)" self.unmapped_platforms[label] = self.unmapped_platforms.get(label, 0) + 1 @@ -856,7 +1019,7 @@ def map_account(self, account: dict, password: Optional[str] = None, mapping = via_pvwa mapping_source = "pvwa-platform" logging.warning( - "Unknown platformId '%s' for account '%s' — resolved via PVWA " + "Unknown platformId '%s' for account '%s' - resolved via PVWA " "platform metadata to %s/%s (port %s). Add it to --platform-map " "to lock in.", platform_id, account.get("name", ""), @@ -870,7 +1033,7 @@ def map_account(self, account: dict, password: Optional[str] = None, mapping_source = "keyword-guess" if platform_id: logging.warning( - "Unknown platformId '%s' for account '%s' — pattern-matched " + "Unknown platformId '%s' for account '%s' - pattern-matched " "to %s/%s (port %s). Add it to --platform-map to lock in.", platform_id, account.get("name", ""), mapping.get("record_type"), mapping.get("protocol") or "n/a", @@ -878,26 +1041,26 @@ def map_account(self, account: dict, password: Optional[str] = None, ) else: logging.debug( - "Empty platformId for account '%s' — pattern-matched to %s/%s.", + "Empty platformId for account '%s' - pattern-matched to %s/%s.", account.get("name", ""), mapping.get("record_type"), mapping.get("protocol") or "n/a", ) else: mapping = dict(FALLBACK_PLATFORM_MAP) - mapping_source = "fallback-ssh" + mapping_source = "fallback-login" if platform_id: logging.warning( - "Unknown platformId '%s' for account '%s' — defaulting to " - "pamMachine/SSH. Use --platform-map to override.", + "Unknown platformId '%s' for account '%s' - importing as " + "a standalone login. Use --platform-map to create a PAM resource.", platform_id, account.get("name", "")) else: - logging.debug("Empty platformId for account '%s' — defaulting to pamMachine/SSH.", + logging.debug("Empty platformId for account '%s' - importing as a standalone login.", account.get("name", "")) - # When the mapping came from a fallback path we trust PVWA's + # When the mapping came from an inferred resolution path we trust PVWA's # per-platform Details endpoint over our static defaults for the - # port. Cheap with caching — one call per unique custom platform. - if mapping_source in ("pvwa-platform", "keyword-guess", "fallback-ssh"): + # port. Cheap with caching - one call per unique custom platform. + if mapping_source in ("pvwa-platform", "keyword-guess"): mapping = self._enrich_port_from_details(platform_id, mapping) record_type = mapping.get("record_type", RECORD_TYPE_PAM_MACHINE) @@ -937,7 +1100,8 @@ def map_account(self, account: dict, password: Optional[str] = None, ) if record_type == RECORD_TYPE_LOGIN: - # BusinessWebsite → login record (not pamMachine) + # Explicit website mappings and unresolved platforms become + # standalone login records, never PAM resources with nested users. record = { "type": RECORD_TYPE_LOGIN, "title": item_name or title, @@ -946,26 +1110,102 @@ def map_account(self, account: dict, password: Optional[str] = None, } if url: record["url"] = url + if mapping_source == "fallback-login": + custom = self._build_unmapped_login_custom_fields(account) + if custom: + record["custom"] = custom return record + if record_type == RECORD_TYPE_PAM_REMOTE_BROWSER: + cloud = (mapping.get("cloud") or "").lower() + defaults = self._cloud_rbi_defaults(cloud) + rbi_url = ( + self._first_prop( + props, "URL", "Url", "url", "ConsoleURL", "ConsoleUrl", + "PortalURL", "PortalUrl", "LoginURL", "LoginUrl", + ) + or address + or defaults.get("url") + or url + ) + if rbi_url and not re.match(r"^https?://", rbi_url, re.IGNORECASE): + rbi_url = f"https://{rbi_url}" + + user_title = f"{login}@{title}" if login else f"user@{title or raw_name}" + user_record = { + "type": "pamUser", + "title": user_title, + "login": login, + "password": password or "", + } + secret_mgmt = account.get("secretManagement", {}) + cpm_enabled = secret_mgmt.get("automaticManagementEnabled", True) + if mapping.get("rotation"): + if cpm_enabled: + platform_sched = self._resolve_platform_schedule(platform_id) + if platform_sched: + schedule = copy.deepcopy(platform_sched) + self.platform_schedule_overrides[platform_id] = ( + self.platform_schedule_overrides.get(platform_id, 0) + 1 + ) + else: + schedule = copy.deepcopy(self._default_rotation_schedule) + else: + schedule = {"type": SCHEDULE_ON_DEMAND} + user_record["rotation_settings"] = { + "rotation": mapping["rotation"], + "enabled": "on" if cpm_enabled else "off", + "schedule": schedule, + } + complexity = self._resolve_platform_password_complexity(platform_id) + if complexity: + user_record["rotation_settings"]["password_complexity"] = complexity + self.platform_complexity_overrides[platform_id] = ( + self.platform_complexity_overrides.get(platform_id, 0) + 1 + ) + if password: + user_record["managed"] = True + self._add_source_metadata_to_user(user_record, account, platform_id) + + resource = { + "type": RECORD_TYPE_PAM_REMOTE_BROWSER, + "title": title or raw_name or f"{cloud.upper()} Console", + "url": rbi_url, + "users": [user_record], + "pam_settings": { + "options": { + "connections": "on", + "remote_browser_isolation": "on", + "graphical_session_recording": "on", + }, + "connection": { + "protocol": "http", + "autofill_credentials": user_title, + "autofill_targets": self._build_cloud_autofill_targets(cloud, rbi_url), + }, + }, + } + metadata_fields = self._resolve_platform_metadata(platform_id) + if metadata_fields: + resource.setdefault("custom", []).extend(copy.deepcopy(metadata_fields)) + account_metadata_fields = self._build_account_metadata_custom_fields(account) + if account_metadata_fields: + resource.setdefault("custom", []).extend(account_metadata_fields) + return resource + if record_type in (RECORD_TYPE_PAM_MACHINE, RECORD_TYPE_PAM_DATABASE): secret_type_check = account.get("secretType", "password").lower() - # No target host and no SSH key material → route to login. A + # No target host and no SSH key material -> route to login. A # pamMachine without a host can never be reached by the gateway, # so the credential is more useful as a standalone login record. # SSH keys keep pamMachine semantics even without an address so # the private_pem_key field is preserved. if not address and secret_type_check != "key": - note = (f"CyberArk platform: {platform_id}\n" - "No address — imported as login (not pamMachine)" - if platform_id else - "CyberArk account had no address — imported as login") return { "type": RECORD_TYPE_LOGIN, "title": title or raw_name, "login": login, "password": password or "", - "notes": note, } # Build pamUser nested inside the resource user_record = { @@ -993,7 +1233,7 @@ def map_account(self, account: dict, password: Optional[str] = None, dn = props.get("DistinguishedName", "") or props.get("distinguishedName", "") if dn: user_record["distinguished_name"] = dn - # Rotation settings — derive from CyberArk secretManagement state + # Rotation settings - derive from CyberArk secretManagement state secret_mgmt = account.get("secretManagement", {}) cpm_enabled = secret_mgmt.get("automaticManagementEnabled", True) if mapping.get("rotation"): @@ -1041,18 +1281,9 @@ def map_account(self, account: dict, password: Optional[str] = None, self.platform_complexity_overrides.get( platform_id_for_sched, 0) + 1 ) - reason = secret_mgmt.get("manualManagementReason", "") - if not cpm_enabled: - existing = user_record.get("notes", "") - line = f"CyberArk CPM disabled: {reason}" - user_record["notes"] = f"{existing}\n{line}".strip() - cpm_status = secret_mgmt.get("status", "") - if cpm_status and cpm_status.lower() == "failure": - existing = user_record.get("notes", "") - line = f"CyberArk CPM status: FAILURE ({reason})" - user_record["notes"] = f"{existing}\n{line}".strip() if password: user_record["managed"] = True + self._add_source_metadata_to_user(user_record, account, platform_id) resource_title = title or address or raw_name resource = { @@ -1062,13 +1293,13 @@ def map_account(self, account: dict, password: Optional[str] = None, "port": str(port) if port else "", "users": [user_record], } - # Map LogonDomain → domain_name on resource (Windows AD domain) + # Map LogonDomain -> domain_name on resource (Windows AD domain) if logon_domain and record_type == RECORD_TYPE_PAM_MACHINE: resource["domain_name"] = logon_domain # Derive operating_system on pamMachine so downstream consumers # (notably ``pam action service add``, which only mounts on Windows # hosts) can dispatch on it. Inferred from platformId keywords, - # fallback to protocol (rdp ⇒ windows, ssh ⇒ linux). + # fallback to protocol (rdp => windows, ssh => linux). if record_type == RECORD_TYPE_PAM_MACHINE: inferred_os = self._infer_operating_system( platform_id, mapping.get("protocol"), @@ -1091,7 +1322,7 @@ def map_account(self, account: dict, password: Optional[str] = None, recording = self._resolve_platform_session_recording( platform_id_for_settings) # Trigger workflows resolution (records unmapped items in - # self.platform_workflow_unmapped — return value not used + # self.platform_workflow_unmapped - return value not used # here because Keeper has no per-resource workflow toggles). self._resolve_platform_workflows(platform_id_for_settings) @@ -1135,14 +1366,25 @@ def map_account(self, account: dict, password: Optional[str] = None, copy.deepcopy(metadata_fields)) return resource - logging.warning('Unsupported record_type "%s" for platform "%s" — account skipped', + logging.warning('Unsupported record_type "%s" for platform "%s" - account skipped', record_type, account.get("platformId", "Unknown")) return None def is_incomplete(self, account: dict) -> Tuple[bool, str]: """Check if a CyberArk account is missing required fields for PAM import.""" + platform_id = account.get("platformId", "") + mapping = self.platform_map.get(platform_id) if platform_id else None + if mapping is None: + mapping = self._resolve_from_platform_metadata(platform_id) if platform_id else None + if mapping is None: + mapping = _guess_platform_mapping(platform_id, account.get("name", "")) + + is_cloud_rbi = ( + isinstance(mapping, dict) + and mapping.get("record_type") == RECORD_TYPE_PAM_REMOTE_BROWSER + ) reasons = [] - if not account.get("address"): + if not account.get("address") and not is_cloud_rbi: reasons.append("missing address/host") if not account.get("userName"): reasons.append("missing userName") @@ -1150,3 +1392,4 @@ def is_incomplete(self, account: dict) -> Tuple[bool, str]: return True, "; ".join(reasons) return False, "" + diff --git a/keepercommander/importer/cyberark/pam/constants.py b/keepercommander/importer/cyberark/pam/constants.py index b366f7a66..2d02fac73 100644 --- a/keepercommander/importer/cyberark/pam/constants.py +++ b/keepercommander/importer/cyberark/pam/constants.py @@ -61,6 +61,11 @@ def reset_system_safes() -> None: # Maximum character length for a single custom metadata field value. MAX_PLATFORM_METADATA_VALUE_LEN = 500 +# Equivalent limits for source-account metadata preserved on standalone login +# records when no CyberArk platform mapping can be resolved. +MAX_ACCOUNT_METADATA_FIELDS = 50 +MAX_ACCOUNT_METADATA_VALUE_LEN = 500 + # Maximum safe name length for Keeper shared folder names MAX_SAFE_NAME_LENGTH = 28 @@ -72,6 +77,7 @@ def reset_system_safes() -> None: RECORD_TYPE_PAM_MACHINE = "pamMachine" RECORD_TYPE_PAM_DATABASE = "pamDatabase" RECORD_TYPE_PAM_DIRECTORY = "pamDirectory" +RECORD_TYPE_PAM_REMOTE_BROWSER = "pamRemoteBrowser" # Rotation schedule type emitted in import JSON / PAM settings SCHEDULE_ON_DEMAND = "on-demand" @@ -122,11 +128,29 @@ def reset_system_safes() -> None: "CheckPointGAIA": {"record_type": RECORD_TYPE_PAM_MACHINE, "rotation": "general", "protocol": "ssh", "port": "22"}, # CyberArk internal — service accounts, import as pamMachine/SSH "CyberArk": {"record_type": RECORD_TYPE_PAM_MACHINE, "rotation": "general", "protocol": "ssh", "port": "22"}, + # Cloud consoles: create an RBI resource plus a companion pamUser. + "AWS": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}, + "AWSAccessKeys": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}, + "AWSAccessKey": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}, + "AmazonWebServices": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}, + "Azure": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}, + "AzureAccessKeys": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}, + "AzureAccessKey": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}, + "MicrosoftAzure": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}, + "GCP": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, + "GCPAccessKeys": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, + "GCPServiceAccount": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, + "GoogleCloud": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, + "GoogleCloudPlatform": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, # Web — login record, NOT pamMachine "BusinessWebsite": {"record_type": RECORD_TYPE_LOGIN, "rotation": None, "protocol": None, "port": None}, } -# Fallback mapping for accounts with empty or unknown platformId +# Fallback mapping for accounts that cannot be resolved through the explicit +# platform map, CyberArk platform metadata, or keyword matching. Without a +# reliable platform mapping there is not enough information to create a PAM +# resource and its nested pamUser safely, so preserve the credential as a +# standalone login record. FALLBACK_PLATFORM_MAP = { - "record_type": RECORD_TYPE_PAM_MACHINE, "rotation": "general", "protocol": "ssh", "port": "22", + "record_type": RECORD_TYPE_LOGIN, "rotation": None, "protocol": None, "port": None, } diff --git a/keepercommander/importer/cyberark/pam/dependents.py b/keepercommander/importer/cyberark/pam/dependents.py index 7259308f2..9f143e051 100644 --- a/keepercommander/importer/cyberark/pam/dependents.py +++ b/keepercommander/importer/cyberark/pam/dependents.py @@ -120,11 +120,28 @@ def resolve_account_dependents(client: 'CyberArkPVWAClient', dep, ("Address", "address", "Host", "host", "MachineAddress", "machineAddress", "TargetAddress", "targetAddress", - "ComputerName", "computerName"), + "ComputerName", "computerName", "LogonDevice", "logonDevice", + "Logon Device", "logon device", "Device", "device", + "Server", "server"), ) or _first_nonempty( props, ("address", "Address", "host", "Host", "machineAddress", - "MachineAddress"), + "MachineAddress", "LogonDevice", "logonDevice", "Logon Device", + "logon device", "Device", "device", "Server", "server"), + )) + machine_refs = _nonempty_values( + dep, + ("Address", "address", "Host", "host", "MachineAddress", + "machineAddress", "TargetAddress", "targetAddress", + "ComputerName", "computerName", "LogonDevice", "logonDevice", + "Logon Device", "logon device", "Device", "device", + "Server", "server"), + ) + machine_refs.extend(_nonempty_values( + props, + ("address", "Address", "host", "Host", "machineAddress", + "MachineAddress", "LogonDevice", "logonDevice", "Logon Device", + "logon device", "Device", "device", "Server", "server"), )) # ``platformId`` (Privilege Cloud) is the most reliable type signal — # it returns concise category codes like ``WinService`` / ``SchedTask`` @@ -152,6 +169,7 @@ def resolve_account_dependents(client: 'CyberArkPVWAClient', continue results.append({ "machine_address": address, + "machine_refs": _dedupe(machine_refs), "service_type": _normalize_dependent_type(raw_type), "raw_type": raw_type, "service_name": name, @@ -198,3 +216,29 @@ def _first_nonempty(source: dict, keys: tuple) -> str: if text: return text return "" + + +def _nonempty_values(source: dict, keys: tuple) -> List[str]: + values: List[str] = [] + if not isinstance(source, dict): + return values + for key in keys: + val = source.get(key) + if val is None: + continue + text = str(val).strip() + if text: + values.append(text) + return values + + +def _dedupe(values: List[str]) -> List[str]: + seen = set() + out: List[str] = [] + for value in values: + key = value.casefold() + if key in seen: + continue + seen.add(key) + out.append(value) + return out diff --git a/keepercommander/importer/cyberark/pam/idempotency.py b/keepercommander/importer/cyberark/pam/idempotency.py index 4851c2188..5097e8232 100644 --- a/keepercommander/importer/cyberark/pam/idempotency.py +++ b/keepercommander/importer/cyberark/pam/idempotency.py @@ -44,6 +44,9 @@ from enum import Enum from typing import Any, Dict, List, Optional, Set, Tuple +from keepercommander.commands.pam_import.nsf_helpers import get_folder_record_uids +from keepercommander.commands.pam_import.record_loader import load_pam_record + # --------------------------------------------------------------------------- # Notes marker @@ -203,9 +206,8 @@ def build_existing_index(params, folder_uids) -> ExistingRecordIndex: # are populated by ``api.sync_down``, but a fresh session without a # sync will have them as ``None`` or empty dicts. Bail early so the # importer falls back to always-create mode instead of crashing. - subfolder_record_cache = getattr(params, "subfolder_record_cache", None) or {} folder_cache = getattr(params, "folder_cache", None) or {} - if not folder_cache: + if not folder_cache and not getattr(params, "nested_share_folders", None): return index # Recursively collect record UIDs from every subfolder. @@ -218,15 +220,19 @@ def build_existing_index(params, folder_uids) -> ExistingRecordIndex: continue visited.add(fuid) index.scanned_folder_uids.add(fuid) - for ruid in (subfolder_record_cache.get(fuid) or set()): + record_uids = get_folder_record_uids(params, fuid) + for ruid in record_uids: all_record_uids.add(ruid) index.folder_by_record[ruid] = fuid folder = folder_cache.get(fuid) for sub_uid in (getattr(folder, "subfolders", []) or []) if folder else []: stack.append(sub_uid) + for child_uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + if (info.get("parent_uid") or None) == fuid and child_uid not in visited: + stack.append(child_uid) for ruid in all_record_uids: - rec = vault.KeeperRecord.load(params, ruid) + rec = load_pam_record(params, ruid) or vault.KeeperRecord.load(params, ruid) if rec is None: continue rtype = getattr(rec, "record_type", "") or "" diff --git a/keepercommander/importer/cyberark/pam/linked_accounts.py b/keepercommander/importer/cyberark/pam/linked_accounts.py index 93e4d3720..d4e040662 100644 --- a/keepercommander/importer/cyberark/pam/linked_accounts.py +++ b/keepercommander/importer/cyberark/pam/linked_accounts.py @@ -59,23 +59,15 @@ def resolve_linked_accounts(client: 'CyberArkPVWAClient', "title": user_title, "login": link_data.get("userName", linked_name), "password": password or "", - "notes": f"CyberArk role: {role_name} account\n" - f"Linked to: {account.get('name', account_id)}\n" - f"Source safe: {linked_safe}", + "custom": [ + {"type": "text", "label": "CyberArk Role", "value": [f"{role_name} account"]}, + {"type": "text", "label": "CyberArk Linked To", "value": [account.get("name", account_id)]}, + {"type": "text", "label": "CyberArk Source Safe", "value": [linked_safe]}, + ], "_ca_role": role_name, # Internal: logon, reconcile, or enable "_ca_id": str(linked_id), # Internal: used by idempotency layer "_ca_safe": linked_safe, # Internal: used by idempotency layer } - # Embed CyberArk identity marker so re-imports can match this - # linked account to the existing Keeper record. The linked - # account has its own CyberArk id distinct from the master - # account it decorates, so we tag with ``linked_id`` (not the - # outer ``account_id``). - try: - from .idempotency import annotate_record_with_marker - annotate_record_with_marker(linked_user, str(linked_id), linked_safe) - except Exception: # noqa: BLE001 — never block linked-account resolution on annotation failure - logging.debug("Failed to annotate linked account %s with CyberArk-ID marker", linked_id) result.append(linked_user) logging.info('Resolved linked %s account: %s', role_name, user_title) diff --git a/keepercommander/importer/cyberark/pam/platform_mapping.py b/keepercommander/importer/cyberark/pam/platform_mapping.py index 0920c16b7..5125684b6 100644 --- a/keepercommander/importer/cyberark/pam/platform_mapping.py +++ b/keepercommander/importer/cyberark/pam/platform_mapping.py @@ -9,8 +9,6 @@ import re from typing import Dict, Optional, Tuple -from .constants import FALLBACK_PLATFORM_MAP - # Keyword → mapping for custom/renamed CyberArk platforms (e.g. "Custom-WinDomain" → RDP). # Matched against platformId and account name; first match wins. Database patterns # are checked before generic Windows/Unix to avoid false positives. @@ -24,6 +22,12 @@ ("mssql", {"record_type": "pamDatabase", "rotation": "general", "protocol": "sql-server", "port": "1433", "database_type": "mssql"}), ("sqlserver", {"record_type": "pamDatabase", "rotation": "general", "protocol": "sql-server", "port": "1433", "database_type": "mssql"}), ("mongo", {"record_type": "pamDatabase", "rotation": "general", "protocol": "mongodb", "port": "27017", "database_type": "mongodb"}), + # Cloud consoles (RBI) + ("aws", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}), + ("amazon", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}), + ("azure", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}), + ("gcp", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}), + ("googlecloud", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}), # Windows (RDP) ("windomain", {"record_type": "pamMachine", "rotation": "general", "protocol": "rdp", "port": "3389"}), ("windows", {"record_type": "pamMachine", "rotation": "general", "protocol": "rdp", "port": "3389"}), @@ -47,7 +51,7 @@ def _guess_platform_mapping(platform_id: str, raw_name: str) -> Optional[dict]: """Map unknown/custom platformIds to a record type via keyword scan. - Returns a mapping dict on match, or None (caller uses FALLBACK_PLATFORM_MAP). + Returns a mapping dict on match, or None (caller imports a standalone login). """ haystack = f"{platform_id or ''}\n{raw_name or ''}".lower() if not haystack.strip(): diff --git a/keepercommander/importer/imp_exp.py b/keepercommander/importer/imp_exp.py index 75914c0df..66e1d3075 100644 --- a/keepercommander/importer/imp_exp.py +++ b/keepercommander/importer/imp_exp.py @@ -617,7 +617,7 @@ def import_teams(params, teams, full_sync=False): # type: (KeeperParams, List[ def import_user_permissions(params, shared_folders, - full_sync=False): # type: (KeeperParams, List[ImportSharedFolder], bool) -> None + full_sync=False, unsafe=False): # type: (KeeperParams, List[ImportSharedFolder], bool) -> None if not shared_folders: return @@ -654,7 +654,7 @@ def import_user_permissions(params, folders = [x for x in folders if x.uid in params.shared_folder_cache] if folders: - permissions = prepare_folder_permission(params, folders, full_sync) + permissions = prepare_folder_permission(params, folders, full_sync, unsafe) if permissions: teams_added = 0 users_added = 0 @@ -1889,32 +1889,46 @@ def prepare_folder_add(params, folders, records, manage_users, manage_records, c is_last = True if digest not in folder_hash: - folder_uid = api.generate_record_uid() - folder_type = 'shared_folder' if is_last else 'user_folder' + existing_by_uid = None + if is_last: + sf_uid = getattr(fol, 'uid', None) + if sf_uid and sf_uid in params.shared_folder_cache and sf_uid in params.folder_cache: + # The import file names an existing shared folder by uid. + # Reuse it in place (no new folder, no move) so any + # records that reference this path resolve to it. + shared_folder_key = params.shared_folder_cache[sf_uid]['shared_folder_key_unencrypted'] + existing_by_uid = sf_uid, 'shared_folder', shared_folder_key + + if existing_by_uid is not None: + folder_uid, folder_type, folder_key = existing_by_uid + folder_hash[digest] = existing_by_uid + else: + folder_uid = api.generate_record_uid() + folder_type = 'shared_folder' if is_last else 'user_folder' - fol_req = folder_pb2.FolderRequest() - fol_req.folderUid = base64.urlsafe_b64decode(folder_uid + '==') - fol_req.folderType = 2 if folder_type == 'shared_folder' else 1 + fol_req = folder_pb2.FolderRequest() + fol_req.folderUid = base64.urlsafe_b64decode(folder_uid + '==') + fol_req.folderType = 2 if folder_type == 'shared_folder' else 1 - if parent_uid: - fol_req.parentFolderUid = base64.urlsafe_b64decode(parent_uid + '==') + if parent_uid: + fol_req.parentFolderUid = base64.urlsafe_b64decode(parent_uid + '==') - folder_key = utils.generate_aes_key() - fol_req.encryptedFolderKey = crypto.encrypt_aes_v1(folder_key, params.data_key) + folder_key = utils.generate_aes_key() + fol_req.encryptedFolderKey = crypto.encrypt_aes_v1(folder_key, params.data_key) - data = {'name': comp} - fol_req.folderData = crypto.encrypt_aes_v1(json.dumps(data).encode('utf-8'), folder_key) + data = {'name': comp} + fol_req.folderData = crypto.encrypt_aes_v1(json.dumps(data).encode('utf-8'), folder_key) - if folder_type == 'shared_folder': - fol_req.sharedFolderFields.encryptedFolderName = \ - crypto.encrypt_aes_v1(comp.encode('utf-8'), folder_key) - fol_req.sharedFolderFields.manageUsers = fol.manage_users or manage_users - fol_req.sharedFolderFields.manageRecords = fol.manage_records or manage_records - fol_req.sharedFolderFields.canEdit = fol.can_edit or can_edit - fol_req.sharedFolderFields.canShare = fol.can_share or can_share + if folder_type == 'shared_folder': + fol_req.sharedFolderFields.encryptedFolderName = \ + crypto.encrypt_aes_v1(comp.encode('utf-8'), folder_key) + fol_req.sharedFolderFields.manageUsers = fol.manage_users or manage_users + fol_req.sharedFolderFields.manageRecords = fol.manage_records or manage_records + fol_req.sharedFolderFields.canEdit = fol.can_edit or can_edit + fol_req.sharedFolderFields.canShare = fol.can_share or can_share - folder_add.append(fol_req) - folder_hash[digest] = folder_uid, folder_type, folder_key if folder_type == 'shared_folder' else None + folder_add.append(fol_req) + folder_hash[digest] = folder_uid, folder_type, folder_key if folder_type == 'shared_folder' else None else: folder_uid, folder_type, folder_key = folder_hash[digest] if is_last: @@ -2572,7 +2586,7 @@ def prepare_record_link(params, records): return record_links -def prepare_folder_permission(params, folders, full_sync): +def prepare_folder_permission(params, folders, full_sync, unsafe=False): # type: (KeeperParams, List[ImportSharedFolder], bool) -> list """Prepare a list of API interactions for changes to folder permissions.""" shared_folder_lookup = {} @@ -2668,6 +2682,9 @@ def prepare_folder_permission(params, folders, full_sync): existing_users.update((x['username'] for x in shared_folder['users'])) if params.user in existing_users: existing_users.remove(params.user) + if unsafe: + # Set user to end of array to be removed last + existing_users.add(params.user) keep_teams = set() keep_users = set() diff --git a/keepercommander/keeper_dag/__version__.py b/keepercommander/keeper_dag/__version__.py index ed7133b36..f00ed5d16 100644 --- a/keepercommander/keeper_dag/__version__.py +++ b/keepercommander/keeper_dag/__version__.py @@ -1 +1 @@ -__version__ = '1.1.11' # pragma: no cover +__version__ = '1.2.0' # pragma: no cover diff --git a/keepercommander/keeper_dag/connection/__init__.py b/keepercommander/keeper_dag/connection/__init__.py index 24ef3ba66..19594a6e9 100644 --- a/keepercommander/keeper_dag/connection/__init__.py +++ b/keepercommander/keeper_dag/connection/__init__.py @@ -10,11 +10,13 @@ import os import time import sys +import json from enum import Enum from pydantic import BaseModel -from typing import Optional, Union, Any, Dict, Tuple, TYPE_CHECKING +from typing import Optional, Union, Any, Dict, Tuple, List, TYPE_CHECKING if TYPE_CHECKING: # pragma: no cover Logger = Union[logging.RootLogger, logging.Logger] + from ..types import JitSettings, ConnectionSettingsBase, AiSettings, Meta # What is this? # If used with Commander, router_abbr_pb2 will interfere with router_pb2. @@ -218,6 +220,8 @@ def payload_and_headers(self, payload: Any) -> Tuple[Union[str, bytes], Dict]: self.logger.debug("payload is protobuf") headers = {'Content-Type': 'application/octet-stream'} payload = encrypt_aes(payload.SerializeToString(), self.transmission_key) + elif isinstance(payload, dict): + payload = json.dumps(payload) else: raise Exception("Cannot determine if the model is pydantic or protobuf.") @@ -468,3 +472,18 @@ def get_leafs(self, error=str(err) ) raise DAGException(f"Could not get leafs: {err}") + + def configure_resource(self, + record: Any, + configuration_record_uid: str, + connection_user_uids: List[str] = [], + admin_user_record_uid: Optional[str] = None, + connection_settings: Optional[ConnectionSettingsBase] = None, + jit_settings: Optional[JitSettings] = None, + ai_settings: Optional[AiSettings] = None, + domain_uid: Optional[str] = None, + meta: Optional[Dict] = None, + update_services: Optional[Meta] = None, + agent: Optional[str] = None): + + raise Exception(f"configure_resource does not exists for {self.__class__.__name__}") diff --git a/keepercommander/keeper_dag/connection/ksm.py b/keepercommander/keeper_dag/connection/ksm.py index b4ad0279f..78480af29 100644 --- a/keepercommander/keeper_dag/connection/ksm.py +++ b/keepercommander/keeper_dag/connection/ksm.py @@ -2,6 +2,10 @@ from . import ConnectionBase from ..utils import value_to_boolean from ..exceptions import DAGException, DAGConnectionException +from ..types import (JitSettings, AiSettings, Meta, ConnectionSettingsBase, NetworkSettings, NetworkResource, + NetworkRotation) +from ..crypto import encrypt_aes +from ..__version__ import __version__ from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import ec @@ -17,10 +21,12 @@ import logging import json +import base64 import os import requests import time -from typing import Union, Optional, Tuple, Dict, Any, TYPE_CHECKING +from pydantic import BaseModel +from typing import Union, Optional, Tuple, Dict, Any, List, TYPE_CHECKING if TYPE_CHECKING: # pragma: no cover from keeper_secrets_manager_core.storage import KeyValueStorage @@ -57,7 +63,7 @@ def __init__(self, self.use_read_protobuf = False if self.use_write_protobuf: self.logger.info("KSM cannot use protobuf for writing to the graph, using JSON.") - self.use_write_protobuf = False + self.use_read_protobuf = False if InMemoryKeyValueStorage.is_base64(config): config = utils.base64_to_string(config) @@ -330,3 +336,432 @@ def rest_call_to_router(self, self.logger.info(f"will retry call after {retry_wait} seconds.") time.sleep(retry_wait) + + def configure_resource(self, + record: Record, + configuration_record_uid: str, + connection_user_uids: Optional[List[str]] = None, + admin_user_record_uid: Optional[str] = None, + connection_settings: Optional[ConnectionSettingsBase] = None, + jit_settings: Optional[JitSettings] = None, + ai_settings: Optional[AiSettings] = None, + domain_uid: Optional[str] = None, + meta: Optional[Meta] = None, + update_services: Optional[bool] = False, + agent: Optional[str] = None): + """ + Configure a resource in the PAM graph. + + After creating a V3 record in the Vault, this method is called to configure that record in hte PAM graph. + + `jit_settings` should be an instance of JitSettings. + It will also take JSON and a python dictionary for that model. + + `connection_settings` should be a child instance of ConnectionSettingsBase. + It will also take JSON and a python dictionary for that model. + + `meta` should be an instance of Meta. + It will also take JSON and a python dictionary for that model. + + `ai_settings` should be an instance of AiSettings. + It will also take JSON and a python dictionary for that model. + + :param record: A pamMachine, pamDatabase, or pamDirectory record. + :param configuration_record_uid: The configuration record uid the resource should be connected to. + :param connection_user_uids: List of user UID connected to this resource. + :param admin_user_record_uid: The user UID of the administrator of the resource. + :param connection_settings: Instance of ConnectionSettings. + :param jit_settings: Instance of JitSettings. + :param ai_settings: Instance of AiSettings. + :param domain_uid: Indicated this resource connects to a domain. + :param meta: Meta information stored on the DATA edge. + :param update_services: Allow rotation to update services on this resource. + :param agent: A custom HTTP agent. Shows up in Data Dog. + :return: + """ + + if record is None: + raise Exception("Record is None. Cannot set_record_rotation.") + + # only configure resources. + if record.type not in ["pamMachine", "pamDatabase", "pamDirectory"]: + raise ValueError(f"Cannot configure resources on {record.uid}. Record type is {record.type}.") + + if jit_settings is not None: + if isinstance(jit_settings, JitSettings): + jit_settings = jit_settings.model_dump_json() + elif isinstance(jit_settings, str): + try: + json.loads(jit_settings) + except Exception as err: + raise ValueError(f"The jit_settings is not JSON: {err}") + elif isinstance(jit_settings, dict): + jit_settings = json.dumps(jit_settings) + else: + raise ValueError("Unknown structure for jit_settings") + jit_settings = base64.b64encode(encrypt_aes(jit_settings.encode(), record.record_key_bytes)).decode() + + if connection_settings is not None: + self.logger.debug("currently, the Vault reads connection_settings from the record, not the graph.") + if isinstance(connection_settings, ConnectionSettingsBase): + connection_settings = connection_settings.model_dump_json() + elif isinstance(connection_settings, str): + try: + json.loads(connection_settings) + except Exception as err: + raise ValueError(f"The connection_settings is not JSON: {err}") + elif isinstance(connection_settings, dict): + connection_settings = json.dumps(connection_settings) + else: + raise ValueError("Unknown structure for connection_settings") + + if meta is not None: + if isinstance(meta, Meta): + meta = meta.model_dump_json() + elif isinstance(meta, str): + try: + json.loads(meta) + except Exception as err: + raise ValueError(f"The meta is not JSON: {err}") + elif isinstance(meta, dict): + meta = json.dumps(meta) + else: + raise ValueError("Unknown structure for meta") + + if ai_settings is not None: + if isinstance(ai_settings, AiSettings): + ai_settings = ai_settings.model_dump_json() + elif isinstance(ai_settings, str): + try: + json.loads(ai_settings) + except Exception as err: + raise ValueError(f"The ai_settings is not JSON: {err}") + elif isinstance(ai_settings, dict): + ai_settings = json.dumps(ai_settings) + else: + raise ValueError("Unknown structure for ai_settings") + ai_settings = base64.b64encode(encrypt_aes(ai_settings.encode(), record.record_key_bytes)).decode() + + if agent is None: + agent = f"keeper-dag/{__version__}" + + endpoint = self._endpoint("configure_resource") + self.logger.debug(f"endpoint {endpoint}") + + try: + payload = dict( + recordUid=record.uid, + networkUid=configuration_record_uid, + adminUid=admin_user_record_uid, + meta=meta, + connectionSettings=connection_settings, + connectUsers=connection_user_uids, + domainUid=domain_uid, + jitSettings=jit_settings, + updateServices=update_services, + keeperAiSettings=ai_settings + ) + + print(json.dumps(payload)) + + payload, headers = self.payload_and_headers(payload) + self.rest_call_to_router(http_method="POST", + endpoint=endpoint, + payload=payload, + headers=headers, + agent=agent) + except Exception as err: + raise DAGException(f"Could not configure resource for {record.uid}: {err}") + + def set_record_rotation(self, + record: Record, + configuration_record_uid: str, + schedule: Optional[dict | str] = None, + resource_uid: Optional[str] = None, + pwd_complexity: Optional[dict] = None, + disabled: bool = False, + remote_address: Optional[str] = None, + noop: bool = False, + saas_configuration_uid: Optional[str] = None, + update_services: bool = False, + service_resource_uids: Optional[List[str]] = None, + service_names: Optional[List] = None, + agent: Optional[str] = None): + + """ + Set the rotation setting on a PAM User record. + + The `schedule` can either be a dictionary or JSON. + The `pwd_complexity` can either be a dictionary or JSON. + + The service names is a list of resource UID and names. + The resource UID should be found in the `service_resource_uids` list. + For example, + + Examples: + service_names = [ + { + "resource_uid": "ztcAMLHaIU_uUK6lYK0GNw", + "names": [ + { + "type": "service", + "items": [{"name": "Service Name", "via_discovery": True}] + } + ] + } + ] + + `names` can also be a list of `UserAclServiceNames` from `discovery-common`. + + :param record: The PAM User record to set rotation settings. + :param configuration_record_uid: The configuration record UID. + :param schedule: If set, password rotation will happen on a schedule. + :param resource_uid: If set, the user will belong to this resource. + :param pwd_complexity: The complexity of the password. + :param disabled: Is True, rotation will not be allowed. + :param remote_address: The IP address of the machine making the request. + :param noop: If True, user account are not updated during rotations. + :param saas_configuration_uid: If set, this user is used on a remote SaaS. + :param update_services: If True, if the user is used for a services, + those services password will also be rotated. + :param service_resource_uids: A list of resource UID where this user is used for a service. + :param service_names: List of dictionaries containing the resource UID, from service_resource_uids, and + and and a base64, encrypted, JSON. + :param agent: A custom HTTP agent. Shows up in Data Dog. + :return: + """ + + if record is None: + raise Exception("Record is None. Cannot set_record_rotation.") + + # Only set the rotation setting on PAM User records. + if record.type != "pamUser": + raise ValueError(f"Cannot set the rotation setting on {record.uid}. Record type is {record.type}.") + + if schedule is None: + schedule = "" + else: + if isinstance(schedule, dict): + schedule = json.dumps(schedule) + if isinstance(schedule, str): + try: + json.loads(schedule) + except Exception as err: + raise ValueError(f"The schedules is not JSON: {err}") + + if pwd_complexity is None: + pwd_complexity = {} + if isinstance(pwd_complexity, dict): + pwd_complexity = json.dumps(pwd_complexity) + if isinstance(pwd_complexity, str): + pwd_complexity = pwd_complexity.encode() + if not isinstance(pwd_complexity, bytes): + raise ValueError("The complexity is not a dictionary, string or bytes.") + pwd_complexity = base64.b64encode(encrypt_aes(pwd_complexity, record.record_key_bytes)).decode() + + if agent is None: + agent = f"keeper-dag/{__version__}" + + endpoint = self._endpoint("set_record_rotation") + self.logger.debug(f"endpoint {endpoint}") + + try: + payload = dict( + recordUid=record.uid, + revision=record.revision, + networkUid=configuration_record_uid, + disabled=disabled, + schedules=schedule, + pwdComplexity=pwd_complexity, + remoteAddress=remote_address, + noop=noop, + updateServices=update_services + ) + if resource_uid is not None: + payload["resourceUid"] = resource_uid + if saas_configuration_uid is not None: + payload["saasConfiguration"] = saas_configuration_uid + if service_resource_uids is not None and isinstance(service_resource_uids, list): + payload["serviceResources"] = service_resource_uids + if service_names is not None: + if not isinstance(service_names, list): + raise ValueError("service_names should be a list of objects that contain the resourceUID and name.") + + payload["serviceNames"] = [] + + for item in service_names: + + name = item.get("names") + if isinstance(name, BaseModel): + name = name.model_dump() + elif isinstance(name, str): + try: + name = json.loads(name) + except (Exception,): + raise ValueError("The name is not JSON.") + elif isinstance(name, list): + name = json.dumps(name) + else: + raise ValueError(f"The name is not a list, string or pydantic model.") + name = base64.b64encode(encrypt_aes(name.encode(), record.record_key_bytes)).decode() + + payload["serviceNames"].append( + { + "names": name, + "resourceUid": item.get("resource_uid") + } + ) + + payload, headers = self.payload_and_headers(payload) + + print(payload) + + self.rest_call_to_router(http_method="POST", + endpoint=endpoint, + payload=payload, + headers=headers, + agent=agent) + except Exception as err: + raise DAGException(f"Could not set the record's rotation setting for {record.uid}: {err}") + + def configure_network_graph(self, + configuration_record_uid: str, + network_settings: Optional[NetworkSettings] = None, + resources: Optional[List[NetworkResource]] = None, + rotations: Optional[List[NetworkRotation]] = None, + agent: Optional[str] = None): + + """ + Bulk configure network/configuraton. + + :param configuration_record_uid: The UID of the configuration record. + :param network_settings: Instance of NetworkSettings + :param resources: + :param rotations: + :param agent: + :return: + """ + + if agent is None: + agent = f"keeper-dag/{__version__}" + + endpoint = self._endpoint("configure_network_graph") + self.logger.debug(f"endpoint {endpoint}") + + try: + payload: Dict[str, Any] = { + "recordUid": configuration_record_uid + } + + if network_settings is not None: + if isinstance(network_settings, NetworkSettings): + network_settings = network_settings.model_dump() + elif isinstance(network_settings, str): + try: + network_settings = json.loads(network_settings) + except Exception as err: + raise ValueError(f"The network_settings is not JSON: {err}") + else: + raise ValueError("Unknown structure for network_settings") + + payload["networkSettings"] = network_settings + + if resources is not None: + + if not isinstance(resources, list): + raise ValueError("resources is not a list") + + encoded_resources = [] + for item in resources: + item.networkUid = configuration_record_uid + + # If we have meta, make sure its JSON + if item.meta: + if isinstance(item.meta, Meta): + item.meta = item.meta.encode() + elif isinstance(item.meta, dict): + item.meta = json.dumps(item.meta) + elif isinstance(item.meta, str): + try: + json.loads(item.meta) + except Exception as err: + raise ValueError(f"The meta for {item.recordUid} is not JSON: {err}") + + # If we have connectionSettings, make sure its JSON + if item.connectionSettings: + if isinstance(item.connectionSettings, ConnectionSettingsBase): + item.connectionSettings = item.connectionSettings.encode() + elif isinstance(item.connectionSettings, dict): + item.connectionSettings = json.dumps(item.connectionSettings) + elif isinstance(item.connectionSettings, str): + try: + json.loads(item.connectionSettings) + except Exception as err: + raise ValueError(f"The connectionSettings for {item.recordUid} is not JSON: {err}") + + # For jitSettings we need the encrypted JSON as Base64. + # We cannot encrypt in here since we don't have the record key bytes. + if item.jitSettings: + if isinstance(item.jitSettings, str): + try: + json.loads(item.jitSettings) + raise ValueError(f"The jitSettings for {item.recordUid} is JSON, " + "should be base64 str.") + except (Exception,): + pass + else: + raise ValueError(f"The jitSettings for {item.recordUid} is not a base64 str.") + + # For keeperAiSettings we need the encrypted JSON as Base64. + # We cannot encrypt in here since we don't have the record key bytes. + if item.keeperAiSettings: + if isinstance(item.keeperAiSettings, str): + try: + json.loads(item.keeperAiSettings) + raise ValueError(f"The keeperAiSettings for {item.recordUid} is JSON, " + "should be base64 str.") + except (Exception,): + pass + else: + raise ValueError(f"The keeperAiSettings for {item.recordUid} is not a base64 str.") + + encoded_resources.append(item.model_dump(exclude_none=True)) + payload["resources"] = encoded_resources + + if rotations is not None: + + if not isinstance(rotations, list): + raise ValueError("rotations is not a list") + + encoded_rotations = [] + for item in rotations: + item.networkUid = configuration_record_uid + + if item.schedule: + if isinstance(item.schedule, str): + try: + json.loads(item.schedule) + except (Exception,): + raise ValueError(f"The schedule for {item.recordUid} is not valid JSON.") + else: + raise ValueError(f"The schedule for {item.recordUid} is not JSON.") + + encoded_rotations.append(item.model_dump(exclude_none=True)) + + payload["rotations"] = encoded_rotations + + payload, headers = self.payload_and_headers(payload) + + print(payload) + + self.rest_call_to_router(http_method="POST", + endpoint=endpoint, + payload=payload, + headers=headers, + agent=agent) + except Exception as err: + raise DAGException(f"Could not configure network graph for {configuration_record_uid}: {err}") + + + + diff --git a/keepercommander/keeper_dag/dag.py b/keepercommander/keeper_dag/dag.py index a9066528e..ff6685a15 100644 --- a/keepercommander/keeper_dag/dag.py +++ b/keepercommander/keeper_dag/dag.py @@ -60,7 +60,9 @@ def __init__(self, data_requires_encryption: bool = False, log_prefix: str = "GraphSync", save_batch_count: Optional[int] = None, + path_aware: bool = False, agent: Optional[str] = None, + dedup_edges: bool = False): """ @@ -85,6 +87,7 @@ def __init__(self, :param data_requires_encryption: Data edges are already encrypted. Default is False. :param log_prefix: Text prepended to the log messages. Handy if dealing with multiple graphs. :param save_batch_count: The number of edges to save at one time. + :param path_aware: Use path when managing edges. :param agent: User Agent to send with web service requests. :param dedup_edges: Remove modified edges if the same edge added before save. :return: Instance of GraphSync @@ -105,6 +108,9 @@ def __init__(self, self.dedup_edge = value_to_boolean(os.environ.get("GS_DEDUP_EDGES", dedup_edges)) self.dedup_edge_warning = value_to_boolean(os.environ.get("GS_DEDUP_EDGES_WARN", False)) + # If True, edges will use their type and the path as a unqiue key. + self.path_aware = path_aware + if self.dedup_edge and auto_save: raise Exception("Cannot run dedup_edge and auto_save at the same time. The dedup_edge feature only works " "in bulk saves.") @@ -703,7 +709,9 @@ def _load(self, sync_point: int = 0): f"edge type {edge_type}", level=3) if edge_type == EdgeType.DELETION: - tail.disconnect_from(head) + tail.disconnect_from(vertex=head, + path=data.path, + path_aware=self.path_aware) else: content = data.content if content is not None: @@ -711,16 +719,18 @@ def _load(self, sync_point: int = 0): content = str_to_bytes(content) # Connect this vertex to the head vertex. It belongs to that head vertex. - tail.belongs_to( - vertex=head, - edge_type=edge_type, - content=content, - # ACL and LINK edges are not encrypted. - is_encrypted=False, - path=data.path, - modified=False, - from_load=True - ) + if tail and head: + tail.belongs_to( + vertex=head, + edge_type=edge_type, + content=content, + # ACL and LINK edges are not encrypted. + is_encrypted=False, + path_aware=self.path_aware, + path=data.path, + modified=False, + from_load=True + ) self.debug("", level=2) self.debug(" PROCESS the DATA edges", level=2) @@ -759,6 +769,7 @@ def _load(self, sync_point: int = 0): content=content, # Assume DATA is encrypted; it might not be but, we will handle that later. is_encrypted=True, + path_aware=self.path_aware, path=data.path, modified=False, from_load=True, diff --git a/keepercommander/keeper_dag/edge.py b/keepercommander/keeper_dag/edge.py index ccc0faaa1..83ba53aa4 100644 --- a/keepercommander/keeper_dag/edge.py +++ b/keepercommander/keeper_dag/edge.py @@ -21,6 +21,7 @@ def __init__(self, head_uid: str, version: int = 0, content: Optional[Any] = None, + path_aware: Optional[bool] = None, path: Optional[str] = None, modified: bool = True, block_content_auto_save: bool = False, @@ -37,7 +38,8 @@ def __init__(self, :param head_uid: The vertex uid that has this edge's vertex. The vertex uid that the edge arrow points at. :param version: Version of this edge. :param content: The content of this edge. - :param path: Short tag about this edge. Do + :param ignore_path: + :param path: Short tag about this edge. :param modified: :param block_content_auto_save: :param from_load: Is this being called from the load() method? @@ -61,6 +63,9 @@ def __init__(self, # We want to only save the newest duplicated edge, so skip prior ones. self.skip_on_save: bool = False + # If an edge need to be filtered, should the path be used? + self.path_aware = path_aware + # Block auto save in the content setter. # When creating an edge, don't save until the edge is added to the edge list. self.block_content_auto_save = block_content_auto_save diff --git a/keepercommander/keeper_dag/types.py b/keepercommander/keeper_dag/types.py index 4b614fbfe..f969909fd 100644 --- a/keepercommander/keeper_dag/types.py +++ b/keepercommander/keeper_dag/types.py @@ -1,7 +1,10 @@ from __future__ import annotations +from .crypto import encrypt_aes from enum import Enum -from pydantic import BaseModel -from typing import List, Optional, Union +import base64 +import json +from pydantic import BaseModel, ConfigDict +from typing import List, Optional, Union, Dict class BaseEnum(Enum): @@ -188,3 +191,351 @@ class DataPayload(BaseModel): origin: Ref dataList: List graphId: Optional[int] = 0 + + +class ConnectionProtocolEnum(BaseEnum): + ssh = "ssh" + rdp = "rdp" + vnc = "vnc" + telnet = "telnet" + http = "http" + sqlserver = "sqlserver" + postgresql = "postgresql" + mysql = "mysql" + + +class ConnectionSecurity(BaseEnum): + any = "any" + nla = "nla" + tls = "tls" + rdp = "rdp" + + +class ConnectionSettingsBase(BaseModel): + protocol: ConnectionProtocolEnum + port: Optional[str] = None + allowSupplyUser: bool = False + userRecords: List[str] = [] + recordingIncludeKeys: bool = False + + def encode(self) -> str: + return self.model_dump_json() + + +class ConnectionSettingsSftp(BaseModel): + enableSftp: bool = True + sftpRootDirectory: Optional[str] = None + sftpServerAliveInterval: Optional[int] = 30 + + +class ConnectionSettingsSsh(ConnectionSettingsBase): + protocol: ConnectionProtocolEnum = ConnectionProtocolEnum.ssh + port: str = "22" + allowSupplyUser: bool = True + disableCopy: bool = False + disablePaste: bool = False + colorScheme: Optional[str] = "white-black" + fontSize: Optional[int] = 12 + scrollback: Optional[int] = 2000 + hostKey: Optional[str] = None + command: Optional[str] = None + sftp: Optional[ConnectionSettingsSftp] = ConnectionSettingsSftp() + + +class ConnectionSettingsRdp(ConnectionSettingsBase): + protocol: ConnectionProtocolEnum = ConnectionProtocolEnum.rdp + port: str = "3389" + disableCopy: bool = False + disablePaste: bool = False + security: ConnectionSecurity = ConnectionSecurity.any + disableAuth: bool = False + ignoreCert: bool = True + loadBalanceInfo: Optional[str] = None + sftp: Optional[ConnectionSettingsSftp] = ConnectionSettingsSftp() + disableAudio: bool = False + resizeMethod: str = "display-update" + enableWallpaper: bool = False + enableFullWindowDrag: bool = False + + +class ConnectionSettingsHttp(ConnectionSettingsBase): + protocol: ConnectionProtocolEnum = ConnectionProtocolEnum.http + port: str = "443" + disableCopy: bool = False + disablePaste: bool = False + + +class JitSettings(BaseModel): + createEphemeral: bool = False + elevate: bool = False + elevationMethod: str = "group" + elevationString: Optional[str] = None + baseDistinguishedName: Optional[str] = None + ephemeralAccountType: Optional[str] = None + + def encode(self, record_key_bytes: bytes) -> str: + return base64.b64encode(encrypt_aes(self.model_dump_json().encode(), record_key_bytes)).decode() + + +class AiSettingsRiskTagItemLog(BaseModel): + date: int + userId: str + action: str + + +class AiSettingsRiskTagItem(BaseModel): + tag: str + auditLog: List[AiSettingsRiskTagItemLog] = [] + + +class AiSettingsRiskTag(BaseModel): + allow: List[AiSettingsRiskTagItem] = [] + deny: List[AiSettingsRiskTagItem] = [] + + +class AiSettingsRiskLevel(BaseModel): + aiSessionTerminate: bool = False + tags: AiSettingsRiskTag = AiSettingsRiskTag() + + +class AiSettingsRiskLevels(BaseModel): + low: AiSettingsRiskLevel = AiSettingsRiskLevel() + medium: AiSettingsRiskLevel = AiSettingsRiskLevel() + high: AiSettingsRiskLevel = AiSettingsRiskLevel() + critical: AiSettingsRiskLevel = AiSettingsRiskLevel() + + +class AiSettings(BaseModel): + version: str = "v1.0.0" + riskLevels: AiSettingsRiskLevels = AiSettingsRiskLevels() + + def encode(self, record_key_bytes: bytes) -> str: + return base64.b64encode(encrypt_aes(self.model_dump_json().encode(), record_key_bytes)).decode() + + +class MetaAllSettings(BaseModel): + remoteBrowserIsolation: bool = False + rotation: bool = False + connections: bool = False + portForwards: bool = False + sessionRecording: bool = False + typescriptRecording: bool = False + aiEnabled: bool = False + aiSessionTerminate: bool = False + + +class Meta(BaseModel): + allowedSettings: MetaAllSettings = MetaAllSettings() + idpConfigUid: Optional[str] = None + rotateOnTermination: bool = False + version: int = 1 + locked: bool = False + no_update_services: bool = False + defaultElevationTime: int = 3600000 + + def encode(self) -> str: + return self.model_dump_json() + + +class PortForward(BaseModel): + port: str + reusePort: bool = False + useSpecifiedLocalPort: bool = False + localPort: Optional[str] = None + + def encode(self) -> str: + return self.model_dump_json() + + +class NetworkSettings(BaseModel): + allowedSettings: MetaAllSettings = MetaAllSettings() + idpConfigUid: Optional[str] = None + adminUid: Optional[str] = None + + def encode(self) -> str: + return self.model_dump_json() + + +class NetworkResource(BaseModel): + model_config = ConfigDict(extra='allow') + + recordUid: str + adminUid: Optional[str] = None + meta: Optional[str] = None + connectionSettings: Optional[str] = None + connectUsers: Optional[List[str]] = None + domainUid: Optional[str] = None + jitSettings: Optional[str] = None + keeperAiSettings: Optional[str] = None + updateServices: Optional[bool] = None + + +class NetworkRotation(BaseModel): + model_config = ConfigDict(extra='allow') + + recordUid: str + revision: int + configurationUid: Optional[str] = None + resourceUid: Optional[str] = None + schedule: Optional[str] = None + pwdComplexity: Optional[str] = None + disabled: Optional[bool] = None + updateServices: Optional[bool] = None + serviceResources: Optional[List[str]] = None + serviceNames: Optional[List[Dict]] = None + + +class PasswordComplexity(BaseModel): + length: int = 20 + caps: int = 1 + lowercase: int = 1 + digits: int = 1 + special: int = 1 + specialChars: str = """!@#$%^?();',.=+[]<>{}-_/\\*&:"`~|""" + + def encode(self, record_key_bytes: bytes) -> str: + return base64.b64encode(encrypt_aes(self.model_dump_json().encode(), record_key_bytes)).decode() + +# https://keeper.atlassian.net/wiki/spaces/EPD/pages/3970793540/Rotation+Schedule + + +class ScheduleDow(BaseEnum): + SUNDAY = "SUNDAY" + MONDAY = "MONDAY" + TUESDAY = "TUESDAY" + WEDNESDAY = "WEDNESDAY" + THURSDAY = "THURSDAY" + FRIDAY = "FRIDAY" + SATURDAY = "SATURDAY" + + +class ScheduleOccurrence(BaseEnum): + FIRST = "FIRST" + SECOND = "SECOND" + THIRD = "THIRD" + FOURTH = "FOURTH" + LAST = "LAST" + + +class ScheduleMonth(BaseEnum): + JANUARY = "JANUARY" + FEBRUARY = "FEBRUARY" + MARCH = "MARCH" + APRIL = "APRIL" + MAY = "MAY" + JUNE = "JUNE" + JULY = "JULY" + AUGUST = "AUGUST" + SEPTEMBER = "SEPTEMBER" + OCTOBER = "OCTOBER" + NOVEMBER = "NOVEMBER" + DECEMBER = "DECEMBER" + + +class Schedule(BaseModel): + type: str = "NA" + + def encode(self) -> str: + return json.dumps([self.model_dump(exclude_none=True)]) + + +class ScheduleCron(Schedule): + type: str = "CRON" + cron: str + tz: Optional[str] = None + + +class ScheduleRunOnce(Schedule): + type: str = "RUN_ONCE" + time: str + tz: Optional[str] = None + + +class ScheduleHourly(Schedule): + type: str = "HOURLY" + minute: Optional[int] = None + second: Optional[int] = None + intervalCount: int = 1 + + +class ScheduleDaily(Schedule): + type: str = "DAILY" + time: str + tz: Optional[str] = None + intervalCount: Optional[int] = None + + +class ScheduleWeekly(Schedule): + type: str = "WEEKLY" + weekday: ScheduleDow + time: str + tz: Optional[str] = None + intervalCount: Optional[int] = None + + +class ScheduleMonthByDay(Schedule): + type: str = "MONTHLY_BY_DAY" + monthDay: int + time: str + tz: Optional[str] = None + intervalCount: Optional[int] = None + + +class ScheduleMonthByWeekday(Schedule): + type: str = "MONTHLY_BY_WEEKDAY" + weekday: ScheduleDow + occurrence: ScheduleOccurrence + time: str + tz: Optional[str] = None + intervalCount: Optional[int] = None + + +class ScheduleYearly(Schedule): + type: str = "YEARLY" + month: ScheduleMonth + monthDay: int + time: str + intervalCount: int = 1 + tz: Optional[str] = None + +#################### + + +class ServiceEnum(BaseEnum): + service = "service" + task = "task" + iis_pool = "iis_pool" + dcom = "dcom" + com = "com" + com_plus = "com_plus" + scom = "scom" + + +class ServiceNameItem(BaseModel): + name: str + + # If this was added via Discovery, this will be True + via_discovery: bool = False + + +class ServiceName(BaseModel): + type: ServiceEnum + items: List[ServiceNameItem] = [] + + +class ServiceResourceName(BaseModel): + resourceUid: str + names: List[ServiceName] + + def encode(self, record_key_bytes: bytes) -> Dict: + + names = [] + for item in self.names: + names.append(item.model_dump(mode='json')) + + print(names) + + return { + "resourceUid": self.resourceUid, + "names": base64.b64encode(encrypt_aes(json.dumps(names).encode(), record_key_bytes)).decode() + } \ No newline at end of file diff --git a/keepercommander/keeper_dag/vertex.py b/keepercommander/keeper_dag/vertex.py index 48c45d0c8..944df6c51 100644 --- a/keepercommander/keeper_dag/vertex.py +++ b/keepercommander/keeper_dag/vertex.py @@ -222,55 +222,107 @@ def uid(self): """ return self._uid - def get_edge(self, vertex: DAGVertex, edge_type: EdgeType) -> DAGEdge: + def get_edge(self, + vertex: DAGVertex, + edge_type: EdgeType, + path_aware: Optional[bool] = None, + path: Optional[str] = None) -> Optional[DAGEdge]: + + """ + Get an edge + :param vertex: The vertex where the tail connects. + :param edge_type: The edge type enumeration. + :param path_aware: If True, use the path as a filter. + :param path: Path to match. None will return edges without a path. + :return: An optional DAGEdge instance. + """ + + if path_aware is None: + path_aware = self.dag.path_aware + high_edge = None high_version = -1 for edge in self.edges: - # Get all the edge point at the same vertex. - # Don't include DATA edges. - if edge.head_uid == vertex.uid and edge.edge_type == edge_type: - if edge.version > high_version: - high_version = edge.version - high_edge = edge + if (edge is not None + and edge.head_uid == vertex.uid + and edge.edge_type == edge_type + and edge.version > high_version + and (not path_aware + or (path is None and edge.path is None) + or (path is not None and path == edge.path))): + high_version = edge.version + high_edge = edge return high_edge - def get_highest_edge_version(self, head_uid: str) -> Tuple[int, Optional[DAGEdge]]: + def get_highest_edge_version(self, + head_uid: str, + path_aware: Optional[bool] = None, + path: Optional[str] = None) -> Tuple[int, Optional[DAGEdge]]: """ - Find the highest DAGEdge version of all edge types. + Find the highest DAGEdge version of all edge types and paths. - :param head_uid: + :param head_uid: Where the edge is towards. + :param path_aware: If True, use the path as a filter. + :param path: Path to match. None will return edges without a path. :return: """ + if path_aware is None: + path_aware = self.dag.path_aware + high_edge = None high_version = -1 for edge in self.edges: - # Get all the edge point at the same vertex. - # Don't include DATA edges. - if edge.head_uid == head_uid: - if edge.version > high_version: - high_edge = edge - high_version = edge.version + if (edge is not None + and edge.head_uid == head_uid + and edge.version > high_version + and (not path_aware or path == edge.path)): + high_edge = edge + high_version = edge.version return high_version, high_edge - def edge_count(self, vertex: DAGVertex, edge_type: EdgeType) -> int: + def edge_count(self, + vertex: DAGVertex, + edge_type: EdgeType, + path_aware: Optional[bool] = None, + path: Optional[str] = None) -> int: """ Get the number of edges between two vertices. :param vertex: :param edge_type: + :param path_aware: + :param path: :return: """ + + if path_aware is None: + path_aware = self.dag.path_aware + count = 0 for edge in self.edges: - if edge.head_uid == vertex.uid and edge.edge_type == edge_type: + if (edge is not None + and edge.head_uid == vertex.uid + and edge.edge_type == edge_type + and (not path_aware or path == edge.path)): count += 1 return count - def edge_by_type(self, vertex: DAGVertex, edge_type: EdgeType) -> List[DAGEdge]: + def edge_by_type(self, + vertex: DAGVertex, + edge_type: EdgeType, + path_aware: Optional[bool] = None, + path: Optional[str] = None) -> List[DAGEdge]: + + if path_aware is None: + path_aware = self.dag.path_aware + edge_list = [] for edge in self.edges: - if edge.edge_type == edge_type and edge.head_uid == vertex.uid: + if (edge is not None + and edge.edge_type == edge_type + and edge.head_uid == vertex.uid + and (not path_aware or path == edge.path)): edge_list.append(edge) return edge_list @@ -278,19 +330,44 @@ def edge_by_type(self, vertex: DAGVertex, edge_type: EdgeType) -> List[DAGEdge]: def has_data(self) -> bool: """ - Does this vertex contain a DATA edge? + Does this vertex contain a DATA edge regardless of path? + + **DEPRECATED** - Use `has_data_for_path` :return: True if vertex has a DATA edge. """ for item in self.edges: - if item.edge_type == EdgeType.DATA: + if item is not None and item.edge_type == EdgeType.DATA: + return True + return False + + def has_data_for_path(self, + path: Optional[str] = None) -> bool: + + """ + Does this vertex contain a DATA edge for the provided path? + + With check if a DAGA edge exists for the path. + If the path is None, it will check if a DATA edge exists where the path is None. + + :param path: Optional path. + :return: True if vertex has a DATA edge for the path. + """ + + for item in self.edges: + if (item is not None + and item.edge_type == EdgeType.DATA + and path == item.path): return True return False - def get_data(self, index: Optional[int] = None) -> Optional[DAGEdge]: + def get_data(self, + path_aware: Optional[bool] = None, + path: Optional[str] = None, + index: Optional[int] = None) -> Optional[DAGEdge]: """ - Get data edge + Get data edge. If the index is None or 0, the latest data edge will be returned. A positive and negative, non-zero, index will return the same data. @@ -299,11 +376,19 @@ def get_data(self, index: Optional[int] = None) -> Optional[DAGEdge]: If there is no data, None is returned. - :param index: + :param path_aware: If True, path will be used as a filter. + :param index: If history allows, get prior data. + :param path: If path_aware is True, find data that matches the path. :return: """ - data_list = self.edge_by_type(self, EdgeType.DATA) + if path_aware is None: + path_aware = self.dag.path_aware + + data_list = self.edge_by_type(self, + edge_type=EdgeType.DATA, + path=path, + path_aware=path_aware) data_count = len(data_list) if data_count == 0: return None @@ -331,6 +416,7 @@ def add_data(self, content: Any, is_encrypted: bool = False, is_serialized: bool = False, + path_aware: Optional[bool] = None, path: Optional[str] = None, modified: bool = True, from_load: bool = False, @@ -342,6 +428,7 @@ def add_data(self, :param content: The content to store in the DATA edge. :param is_encrypted: Is the content encrypted? :param is_serialized: Is the content base64 serialized? + :param path_aware: If True, path will be used as a filter. :param path: Simple string tag to identify the edge. :param modified: Does this modify the content? By default, adding a DATA edge will flag that the edge has been modified. @@ -354,6 +441,9 @@ def add_data(self, self.debug(f"connect {self.uid} to DATA edge", level=1) + if path_aware is None: + path_aware = self.dag.path_aware + # Are we trying to add DATA to a deleted vertex? if not self.active: @@ -384,7 +474,7 @@ def add_data(self, # Get the prior data, set the version and inactive the prior data. version = 0 - prior_data = self.get_data() + prior_data = self.get_data(path=path, path_aware=path_aware) if prior_data is not None: version = prior_data.version + 1 prior_data.active = False @@ -406,6 +496,7 @@ def add_data(self, head_uid=self.uid, version=version, content=content, + path_aware=path_aware, path=path, modified=modified, is_serialized=is_serialized, @@ -418,7 +509,7 @@ def add_data(self, # The history level is per edge type. # It's FIFO, so we will remove the first edge type if we exceed the history level. if self.dag.history_level > 0: - data_count = self.data_count() + data_count = self.data_count(path=path, path_aware=path_aware) while data_count > self.dag.history_level: for index in range(0, len(self.edges) - 1): if self.edges[index].edge_type == EdgeType.DATA: @@ -428,14 +519,29 @@ def add_data(self, self.dag.do_auto_save() - def data_count(self): - return self.edge_count(self, EdgeType.DATA) + def data_count(self, + path_aware: Optional[bool] = None, + path: Optional[str] = None): + + return self.edge_count(self, + path_aware=path_aware, + path=path, + edge_type=EdgeType.DATA) - def data_delete(self): + def data_delete(self, + path_aware: Optional[bool] = None, + path: Optional[str] = None): + + if path_aware is None: + path_aware = self.dag.path_aware # Get the DATA edge. # It will be a reference to itself. - data_edge = self.get_edge(self, EdgeType.DATA) + data_edge = self.get_edge(self, + edge_type=EdgeType.DATA, + path=path, + path_aware=path_aware) + if data_edge is None: self.debug("cannot delete the data, no data edge exists.") @@ -443,26 +549,61 @@ def data_delete(self): self.belongs_to( vertex=self, - edge_type=EdgeType.DELETION + edge_type=EdgeType.DELETION, + path_aware=path_aware, + path=path ) - self.debug(f"deleted data edge for {self.uid}") + self.debug(f"deleted data edge for {self.uid}; path {path}") @property - def latest_data_version(self): + def latest_data_version(self, + path_aware: Optional[bool] = None, + path: Optional[str] = None): + """ + Get the latest DATA edge version. + + :param path_aware: + :param path: + :return: + """ + + if path_aware is None: + path_aware = self.dag.path_aware + version = -1 for edge in self.edges: - if edge.edge_type == EdgeType.DATA and edge.version > version: + # If edge is defined; and the paths match, and the edge if a DATA type; and the edge version is > + if (edge is not None + and (not path_aware or path == edge.path) + and edge.edge_type == EdgeType.DATA and edge.version > version): version = edge.version return version @property def content(self) -> Optional[Union[str, bytes]]: """ - Get the content of the active DATA edge. + Get the content of the first found, active DATA edge for any path. + + If the content is a str, then the content is encrypted. + + **DEPRECATED**: use `content_for_path` + """ + data_edge = self.get_data(path=None, path_aware=False) + if data_edge is None: + return None + return data_edge.content + + def content_for_path(self, + path: Optional[str] = None) -> Optional[Union[str, bytes]]: + """ + Get the content of the active DATA edge for a path. If the content is a str, then the content is encrypted. + + :param path: Get content for a specific path. + :return: Content as a str or bytes. """ - data_edge = self.get_data() + data_edge = self.get_data(path=path, path_aware=self.dag.path_aware) if data_edge is None: return None return data_edge.content @@ -470,10 +611,25 @@ def content(self) -> Optional[Union[str, bytes]]: @property def content_as_dict(self) -> Optional[dict]: """ - Get the content from the active DATA edge as a dictionary. + Get the content from the first found, active DATA edge, for any path, as a dictionary. :return: Content as a dictionary. + + **DEPRECATED**: use `content_as_dict_for_path` """ - data_edge = self.get_data() + data_edge = self.get_data(path=None, path_aware=False) + if data_edge is None: + return None + return data_edge.content_as_dict + + def content_as_dict_for_path(self, + path: Optional[str] = None) -> Optional[dict]: + """ + Get the content from the active DATA edge, for a path, as a dictionary. + + :param path: Get content for a specific path. + :return: Content as a dictionary. + """ + data_edge = self.get_data(path=path, path_aware=self.dag.path_aware) if data_edge is None: return None return data_edge.content_as_dict @@ -481,23 +637,47 @@ def content_as_dict(self) -> Optional[dict]: @property def content_as_str(self) -> Optional[str]: """ - Get the content from the active DATA edge as a str. + Get the content from the first found, active DATA edge, for any path, as a str. + + **DEPRECATED**: use `content_as_str_for_path` + + :return: Content as a str. + """ + + data_edge = self.get_data(path=None, path_aware=False) + if data_edge is None: + return None + return data_edge.content_as_str + + def content_as_str_for_path(self, + path: Optional[str] = None) -> Optional[str]: + """ + Get the content from the active DATA edge, with no path, as a str. :return: Content as a str. """ - data_edge = self.get_data() + data_edge = self.get_data(path=path, path_aware=self.dag.path_aware) if data_edge is None: return None return data_edge.content_as_str - def content_as_object(self, meta_class: Type[T]) -> Optional[T]: + def content_as_object(self, + meta_class: Type[T], + path_aware: Optional[bool] = False, + path: Optional[str] = None) -> Optional[T]: """ Get the content as a pydantic based object. :param meta_class: The class to return + :param path_aware: If True, path will be used as a filter. + :param path: Find a DATA edge that matches this path. None with match None. :return: """ - data_edge = self.get_data() + + if path_aware is None: + path_aware = self.dag.path_aware + + data_edge = self.get_data(path=path, path_aware=path_aware) if data_edge is None: return None @@ -507,13 +687,19 @@ def content_as_object(self, meta_class: Type[T]) -> Optional[T]: def has_key(self) -> bool: """ - Does this vertex contain any KEY or ACL edges? + Does this vertex contain any KEY? + + If `path_aware` is enabled, the KEY must not have a path. + If not enabled, it will take the first path it finds regardless of path. + This will be the active KEY. - :return: True if vertex has a KEY or ACL edge. + :return: True if vertex has a KEY edge where the path is None. """ for item in self.edges: - if item.edge_type == EdgeType.KEY: + if (item + and item.edge_type == EdgeType.KEY + and (not self.dag.path_aware or item.path is None)): return True return False @@ -522,6 +708,7 @@ def belongs_to(self, edge_type: EdgeType, content: Optional[Any] = None, is_encrypted: bool = False, + path_aware: Optional[bool] = None, path: Optional[str] = None, modified: bool = True, from_load: bool = False): @@ -539,6 +726,7 @@ def belongs_to(self, :param edge_type: The edge type that connects the two vertices. :param content: Data to store as the edges content. :param is_encrypted: Is the content encrypted? + :param path_aware: If True, the path will be used as a filter. :param path: Text tag for the edge. :param modified: Does adding this edge modify the stored DAG? :param from_load: Is being connected from load() method? @@ -547,8 +735,12 @@ def belongs_to(self, self.debug(f"connect {self.uid} to {vertex.uid} with edge type {edge_type.value}", level=1) + if path_aware is None: + path_aware = self.dag.path_aware + if vertex is None: raise ValueError("Vertex is blank.") + if self.uid == self.dag.uid and not (edge_type == EdgeType.DATA or edge_type == EdgeType.DELETION): if not from_load: raise DAGIllegalEdgeException(f"Cannot create edge to self for edge type {edge_type}.") @@ -568,13 +760,18 @@ def belongs_to(self, # Figure out what version of the edge we are. - version, version_edge = self.get_highest_edge_version(head_uid=vertex.uid) + version, version_edge = self.get_highest_edge_version(head_uid=str(vertex.uid), + path=path, + path_aware=path_aware) # If the new edge is not DELETION if edge_type != EdgeType.DELETION: # Find the current active edge for this edge type to make it inactive. - current_edge_by_type = self.get_edge(vertex, edge_type) + current_edge_by_type = self.get_edge(vertex=vertex, + edge_type=edge_type, + path_aware=path_aware, + path=path) if current_edge_by_type is not None: current_edge_by_type.active = False @@ -589,7 +786,10 @@ def belongs_to(self, self.dag.debug_stacktrace() # If we are adding a non-DELETION edge, it will inactivate the DELETION edge. - highest_deletion_edge = self.get_edge(vertex, EdgeType.DELETION) + highest_deletion_edge = self.get_edge(vertex=vertex, + edge_type=EdgeType.DELETION, + path_aware=path_aware, + path=path) if highest_deletion_edge is not None: highest_deletion_edge.active = False @@ -604,7 +804,7 @@ def belongs_to(self, if edge_type == EdgeType.DELETION: return - if self.dag.dedup_edge and version_edge.modified: + if self.dag.dedup_edge and version_edge is not None and version_edge.modified: version_edge.skip_on_save = True if self.dag.dedup_edge_warning: self.dag.debug("edge was deleted in session, will not save DELETION edge") @@ -624,6 +824,7 @@ def belongs_to(self, block_content_auto_save=True, content=content, is_encrypted=is_encrypted, + path_aware=path_aware, path=path, modified=modified ) @@ -637,18 +838,27 @@ def belongs_to(self, def belongs_to_root(self, edge_type: EdgeType, - path: Optional[str] = None): + path_aware: Optional[bool] = None, + path: Optional[str] = None, + content: Optional[Any] = None, + is_encrypted: bool = False): """ Connect the vertex to the root vertex. :param edge_type: The type of edge to use for the connection. + :param path_aware: If True, the path will be used as a filter. :param path: Short tag for this edge. + :param content: Data to store as the edges content. + :param is_encrypted: Is the content encrypted? :return: """ self.debug(f"connect {self.uid} to root", level=1) + if path_aware is None: + path_aware = self.dag.path_aware + if self.uid == self.dag.uid: raise DAGIllegalEdgeException("Cannot create edge to self.") @@ -657,16 +867,29 @@ def belongs_to_root(self, # We are adding the root, we can enable auto save now. # We can get the correct stream id with an edge to the root vertex. - self.belongs_to(self.dag.get_root, edge_type=edge_type, path=path) - - self.dag.allow_auto_save = True - self.dag.do_auto_save() - - def has_vertices(self, edge_type: Optional[EdgeType] = None, allow_inactive: bool = False, + root = self.dag.get_root + if root: + self.belongs_to(vertex=root, + edge_type=edge_type, + path=path, + path_aware=path_aware, + content=content, + is_encrypted=is_encrypted) + + self.dag.allow_auto_save = True + self.dag.do_auto_save() + + def has_vertices(self, + edge_type: Optional[EdgeType] = None, + allow_inactive: bool = False, allow_self_ref: bool = False) -> List[DAGVertex]: """ Get a list of vertices that belong to this vertex. + + :param edge_type: Filter for a specific edge type. + :param allow_inactive: If True, include vertices that are inactive. + :param allow_self_ref: If True, allow vertices that refer to themselves. :return: List of DAGVertex """ @@ -679,23 +902,26 @@ def has_vertices(self, edge_type: Optional[EdgeType] = None, allow_inactive: boo continue vertex = self.dag.get_vertex(uid) - if edge_type is not None: - edge = vertex.get_edge(self, edge_type=edge_type) - if edge is not None: - vertices.append(vertex) - - # If no edge type was specified, do not return DATA and DELETION. - # Also do not include vertices that are inactive by default. - elif edge_type != EdgeType.DATA and edge_type != EdgeType.DELETION: - if vertex.active is True or allow_inactive is True: - vertices.append(vertex) + if vertex: + if edge_type is not None: + edge = vertex.get_edge(self, edge_type=edge_type) + if edge is not None: + vertices.append(vertex) + + # If no edge type was specified, do not return DATA and DELETION. + # Also do not include vertices that are inactive by default. + elif edge_type != EdgeType.DATA and edge_type != EdgeType.DELETION: + if vertex.active is True or allow_inactive is True: + vertices.append(vertex) return vertices - def has(self, vertex: DAGVertex, edge_type: Optional[EdgeType] = None) -> bool: + def has(self, + vertex: DAGVertex, + edge_type: Optional[EdgeType] = None) -> bool: """ - Does this vertex have the passed in vertex? + Does "self" have the vertex passed in? :return: True if request vertex belongs to this vertex. False if it does not. @@ -713,12 +939,12 @@ def belongs_to_vertices(self) -> List[DAGVertex]: vertices = [] for edge in self.edges: # If the edge is not a DATA or DELETION type, and the edge is the highest version/active - if edge.edge_type != EdgeType.DATA and edge.edge_type != EdgeType.DELETION and edge.active is True: + if edge and edge.edge_type != EdgeType.DATA and edge.edge_type != EdgeType.DELETION and edge.active is True: # The head will point at the remote vertex. # If it is active, and not already in the list, add it to the list of vertices this vertex belongs to. vertex = self.dag.get_vertex(edge.head_uid) - if vertex.active is True and vertex not in vertices: + if vertex and vertex.active is True and vertex not in vertices: vertices.append(vertex) return vertices @@ -736,7 +962,10 @@ def belongs_to_a_vertex(self) -> bool: return len(self.belongs_to_vertices()) > 0 - def disconnect_from(self, vertex: DAGVertex, path: Optional[str] = None): + def disconnect_from(self, + vertex: DAGVertex, + path_aware: Optional[bool] = None, + path: Optional[str] = None): """ Disconnect this vertex from another vertex. @@ -745,6 +974,7 @@ def disconnect_from(self, vertex: DAGVertex, path: Optional[str] = None): If the vertex no longer belongs to another vertex, the vertex will be deleted. :param vertex: The vertex this vertex belongs to + :param path_aware: If True, the path will be used a specific edge. :param path: an Optional path for the DELETION edge. :return: """ @@ -752,15 +982,21 @@ def disconnect_from(self, vertex: DAGVertex, path: Optional[str] = None): if vertex is None: raise ValueError("Vertex is blank.") + if path_aware is None: + path_aware = self.dag.path_aware + # Flag all the edges as inactive. for edge in self.edges: - if edge.head_uid == vertex.uid and edge.edge_type: + if (edge is not None + and (not path_aware or edge.path == path) + and edge.head_uid == vertex.uid): edge.active = False # Add the DELETION edge self.belongs_to( vertex=vertex, edge_type=EdgeType.DELETION, + path_aware=path_aware, path=path ) @@ -768,12 +1004,17 @@ def disconnect_from(self, vertex: DAGVertex, path: Optional[str] = None): # There is no longer a KEY edge to decrypt the DATA. has_active_key_edge = False for edge in self.edges: - if edge.edge_type == EdgeType.KEY and edge.active is True: + if (edge is not None + and (not path_aware or path == edge.path) + and edge.edge_type == EdgeType.KEY + and edge.active is True): has_active_key_edge = True break if not has_active_key_edge: for edge in self.edges: - if edge.edge_type == EdgeType.DATA: + if (edge is not None + and (not path_aware or path == edge.path) + and edge.edge_type == EdgeType.DATA): edge.active = False if not self.belongs_to_a_vertex: @@ -875,7 +1116,7 @@ def walk_down_path(self, path: Union[str, List[str]]) -> Optional[DAGVertex]: self.debug(f"vertex {self.uid} has {vertex.uid}", level=2) for edge in vertex.edges: # If the edge matches the current path, the head of the edge is this vertex, a route exists. - if edge.path == current_path and edge.head_uid == self.uid: + if edge and edge.path == current_path and edge.head_uid == self.uid: # If there is no path left, this is our vertex if len(path) == 0: return vertex @@ -893,7 +1134,7 @@ def get_paths(self) -> List[str]: paths = [] for vertex in self.has_vertices(): for edge in vertex.edges: - if edge.path is None or edge.path == "": + if edge and edge.path is None or edge.path == "": continue paths.append(edge.path) diff --git a/keepercommander/sanitization.py b/keepercommander/sanitization.py new file mode 100644 index 000000000..22df619a6 --- /dev/null +++ b/keepercommander/sanitization.py @@ -0,0 +1,71 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' Union[str, list, dict]: + """Mask a Keeper record field's `value`, preserving its container shape.""" + if isinstance(value, list): + return ['***' for _ in value] + if isinstance(value, dict): + return {k: '***' for k in value} + return '***' + + +def sanitize_nested_data(data: Any) -> Any: + """Recursively sanitize nested data structures for logging.""" + if isinstance(data, dict): + field_type = data.get('type') + if isinstance(field_type, str) and field_type.lower() in SENSITIVE_FIELD_TYPES and 'value' in data: + sanitized = dict(data) + sanitized['value'] = mask_field_value(data['value']) + return sanitized + + sanitized = {} + for key, value in data.items(): + if key.lower() in SENSITIVE_DICT_KEYS: + if isinstance(value, str) and len(value) > 0: + sanitized[key] = '*' * min(len(value), 15) + else: + sanitized[key] = '***' + else: + sanitized[key] = sanitize_nested_data(value) + return sanitized + elif isinstance(data, list): + return [sanitize_nested_data(item) for item in data] + else: + return data + + +def sanitize_protobuf_json(json_str: str) -> str: + """Sanitize sensitive data from protobuf JSON before logging.""" + try: + data = json.loads(json_str) + sanitized = sanitize_nested_data(data) + return json.dumps(sanitized) + except (json.JSONDecodeError, TypeError): + return json_str diff --git a/keepercommander/service/README.md b/keepercommander/service/README.md index 18b8e4e28..a04b01174 100644 --- a/keepercommander/service/README.md +++ b/keepercommander/service/README.md @@ -502,6 +502,16 @@ This automates the complete setup for Slack App integration: The command generates a complete `docker-compose.yml` with both Commander service and Slack App service configured. +**Generated compose environment for the Slack service:** + +| Env var | Value | +|---------|-------| +| `KSM_CONFIG` | Base64 KSM config | +| `COMMANDER_RECORD` | Commander Docker config record UID | +| `SLACK_RECORD` | Slack config record UID | + +Image name used in compose: `keeper/slack-app:latest`. + ### Google Chat App Integration Setup For integrating Commander Service Mode with Google Chat, use the `gchat-app-setup` command: diff --git a/keepercommander/service/commands/create_service.py b/keepercommander/service/commands/create_service.py index d035fde91..8ddc5538d 100644 --- a/keepercommander/service/commands/create_service.py +++ b/keepercommander/service/commands/create_service.py @@ -106,6 +106,10 @@ def execute(self, params: KeeperParams, **kwargs) -> None: from .integrations.sailpoint.service import SailPointService SailPointService.maybe_enable(params, args) + # Re-sanitize commands against each integration's own allowlist in case docker-compose.yml is stale/hand-edited. + from .integrations.runtime_policy import apply_runtime_command_policy + apply_runtime_command_policy(args) + from .integrations.vault_metadata import get_existing_api_key, write_service_metadata existing_api_key = ( get_existing_api_key(params, args.update_vault_record) diff --git a/keepercommander/service/commands/integrations/command_policy.py b/keepercommander/service/commands/integrations/command_policy.py new file mode 100644 index 000000000..f9e84d351 --- /dev/null +++ b/keepercommander/service/commands/integrations/command_policy.py @@ -0,0 +1,41 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' str: + """Normalizes `commands` to exactly the allowed set minus banned; cannot narrow below the allowlist, only add missing/drop extra entries.""" + allowed = [c.strip() for c in allowed if c and c.strip()] + allowed_set = {c.lower() for c in allowed} + banned_set = {c.strip().lower() for c in banned} + filtered = [ + cmd for raw in (commands or '').split(',') + if (cmd := raw.strip()) + and (key := cmd.lower()) not in banned_set + and key in allowed_set + ] + # Input order first, then any missing required allowlist entries. + by_key = {cmd.lower(): cmd for cmd in filtered} + for cmd in allowed: + key = cmd.strip().lower() + if key not in banned_set and key not in by_key: + by_key[key] = cmd.strip() + return ','.join(by_key.values()) + + +def default_allowlist(allowed: Iterable[str], banned: Iterable[str] = ()) -> str: + allowed = list(allowed) + return sanitize_commands(','.join(allowed), allowed, banned) diff --git a/keepercommander/service/commands/integrations/integration_setup_base.py b/keepercommander/service/commands/integrations/integration_setup_base.py index 3d26a5169..901b82c09 100644 --- a/keepercommander/service/commands/integrations/integration_setup_base.py +++ b/keepercommander/service/commands/integrations/integration_setup_base.py @@ -30,6 +30,7 @@ ) from .approvals_setup import ApprovalsChannelProfile, collect_approvals_config, is_valid_keeper_uid from .approvals_sync import merge_approvals_custom_fields, run_approvals_sync_down +from .command_policy import sanitize_commands UUID_PATTERN = re.compile( r'^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' @@ -121,6 +122,15 @@ def get_service_commands(self) -> str: commands = f'{commands},{self.get_command_name()}' return commands + def get_banned_commands(self) -> tuple: + """Commands to always strip from this integration's command list; override per integration if needed.""" + return () + + def sanitize_service_commands(self, commands: str) -> str: + """Restrict `commands` to this integration's own get_service_commands() list.""" + allowed = self.get_service_commands().split(',') + return sanitize_commands(commands, allowed, self.get_banned_commands()) + # -- Parser (auto-built from name, cached per subclass) ---------- def get_parser(self): @@ -456,10 +466,13 @@ def _update_docker_compose(self, setup_result: SetupResult, ) try: + cfg = asdict(service_config) + cfg['commands'] = self.sanitize_service_commands(cfg.get('commands') or '') builder = DockerComposeBuilder( - setup_result, asdict(service_config), + setup_result, cfg, commander_service_name=self.get_commander_service_name(), - commander_container_name=self.get_commander_container_name() + commander_container_name=self.get_commander_container_name(), + commander_environment={self.get_record_env_key(): record_uid}, ) yaml_content = builder.add_integration_service( service_name=service_name, diff --git a/keepercommander/service/commands/integrations/runtime_policy.py b/keepercommander/service/commands/integrations/runtime_policy.py new file mode 100644 index 000000000..e773c4828 --- /dev/null +++ b/keepercommander/service/commands/integrations/runtime_policy.py @@ -0,0 +1,70 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' None: + if getattr(args, 'commands', None) is None: + return + + sanitizers = _integration_sanitizers() + matched = [env_key for env_key in sanitizers if (os.environ.get(env_key) or '').strip()] + if not matched: + return + if len(matched) > 1: + raise ValidationError( + f'Multiple integration env vars are set ({", ".join(matched)}); ' + f'remove all but one before starting the service.' + ) + + env_key = matched[0] + cleaned = sanitizers[env_key](args.commands) + if cleaned != args.commands: + print( + f'Service Mode ({env_key}): removed commands outside the integration ' + f'allowlist before service-create.\n Was: {args.commands}\n Now: {cleaned}' + ) + args.commands = cleaned + + +def _integration_sanitizers(): + """Maps each integration's env var to its own sanitize_service_commands(); SailPoint is included too as a fallback in case SailPointService.maybe_enable() skipped sanitizing.""" + from ..terraform_app_setup import TerraformSetupConstants + from .command_policy import sanitize_commands + from .gchat_app_setup import GChatAppSetupCommand + from .sailpoint_app_setup import SailPointAppSetupCommand + from .slack_app_setup import SlackAppSetupCommand + from .teams_app_setup import TeamsAppSetupCommand + from ...decorators.min_commander_version import TERRAFORM_DOCKER_ENV, TERRAFORM_DOCKER_ENV_LEGACY + + slack = SlackAppSetupCommand() + teams = TeamsAppSetupCommand() + gchat = GChatAppSetupCommand() + sailpoint = SailPointAppSetupCommand() + terraform_sanitizer = lambda commands: sanitize_commands( + commands, TerraformSetupConstants.SERVICE_COMMANDS_LIST + ) + + return { + slack.get_record_env_key(): slack.sanitize_service_commands, + teams.get_record_env_key(): teams.sanitize_service_commands, + gchat.get_record_env_key(): gchat.sanitize_service_commands, + sailpoint.get_record_env_key(): sailpoint.sanitize_service_commands, + TERRAFORM_DOCKER_ENV: terraform_sanitizer, + TERRAFORM_DOCKER_ENV_LEGACY: terraform_sanitizer, + } diff --git a/keepercommander/service/commands/integrations/sailpoint/command_hook.py b/keepercommander/service/commands/integrations/sailpoint/command_hook.py index 30783a1ee..b079514bf 100644 --- a/keepercommander/service/commands/integrations/sailpoint/command_hook.py +++ b/keepercommander/service/commands/integrations/sailpoint/command_hook.py @@ -81,7 +81,7 @@ def before_command(self, params: KeeperParams, command: str) -> Tuple[str, Optio mutation = SailPointCommandParser.parse_identity_mutation(command) if mutation: - err = self._first_scim_identity_error(params, mutation.emails) + err = self._first_scim_identity_error(params, mutation.identifiers) if err: return self._reject(command, err, 403) return command, None diff --git a/keepercommander/service/commands/integrations/sailpoint/command_parse.py b/keepercommander/service/commands/integrations/sailpoint/command_parse.py index a90b22b62..9de73cd92 100644 --- a/keepercommander/service/commands/integrations/sailpoint/command_parse.py +++ b/keepercommander/service/commands/integrations/sailpoint/command_parse.py @@ -63,7 +63,7 @@ def is_grant(self) -> bool: class ParsedIdentityMutation: """enterprise-user identity change (role/team/node) — used for SCIM coexistence.""" - emails: List[str] = field(default_factory=list) + identifiers: List[str] = field(default_factory=list) has_role_change: bool = False has_team_change: bool = False has_node_change: bool = False @@ -134,14 +134,15 @@ def parse_identity_mutation(cls, command: str) -> Optional[ParsedIdentityMutatio if not parsed: return None ns, _unknown = parsed - emails = [e for e in (ns.email or []) if isinstance(e, str) and '@' in e] + # No '@' filter here (unlike parse_invite/parse_transfer): the SCIM guard must see IDs and '@all' too. + identifiers = [e for e in (ns.email or []) if isinstance(e, str) and e.strip()] has_role = bool(ns.add_role or ns.remove_role) has_team = bool(ns.add_team or ns.remove_team) has_node = bool(ns.node) - if not (has_role or has_team or has_node) or not emails: + if not (has_role or has_team or has_node) or not identifiers: return None return ParsedIdentityMutation( - emails=emails, + identifiers=identifiers, has_role_change=has_role, has_team_change=has_team, has_node_change=has_node, diff --git a/keepercommander/service/commands/integrations/sailpoint/command_policy.py b/keepercommander/service/commands/integrations/sailpoint/command_policy.py index 88ec51451..13afa6b99 100644 --- a/keepercommander/service/commands/integrations/sailpoint/command_policy.py +++ b/keepercommander/service/commands/integrations/sailpoint/command_policy.py @@ -17,6 +17,7 @@ from typing import Any, Optional, Tuple from .....utils import is_email +from .. import command_policy as _shared_policy from .command_parse import SailPointCommandParser from .constants import SAILPOINT_ALLOWED_COMMANDS, SAILPOINT_BANNED_COMMANDS @@ -50,25 +51,13 @@ def sanitize(cls, commands: str) -> str: commands are not dropped when the input list is a partial or older compose allowlist. """ - allowed = {c.strip().lower() for c in SAILPOINT_ALLOWED_COMMANDS} - banned = {c.lower() for c in SAILPOINT_BANNED_COMMANDS} - filtered = [ - cmd for raw in (commands or '').split(',') - if (cmd := raw.strip()) - and (key := cmd.lower()) not in banned - and key in allowed - ] - # Input order first, then any missing required allowlist entries. - by_key = {cmd.lower(): cmd for cmd in filtered} - for cmd in SAILPOINT_ALLOWED_COMMANDS: - key = cmd.lower() - if key not in banned and key not in by_key: - by_key[key] = cmd - return ','.join(by_key.values()) + return _shared_policy.sanitize_commands( + commands, SAILPOINT_ALLOWED_COMMANDS, SAILPOINT_BANNED_COMMANDS + ) @classmethod def default_allowlist(cls) -> str: - return cls.sanitize(','.join(SAILPOINT_ALLOWED_COMMANDS)) + return _shared_policy.default_allowlist(SAILPOINT_ALLOWED_COMMANDS, SAILPOINT_BANNED_COMMANDS) @classmethod def validate_enterprise_user(cls, command: str) -> Optional[str]: diff --git a/keepercommander/service/commands/integrations/sailpoint/scim_guard.py b/keepercommander/service/commands/integrations/sailpoint/scim_guard.py index 0e2a42812..b774fc192 100644 --- a/keepercommander/service/commands/integrations/sailpoint/scim_guard.py +++ b/keepercommander/service/commands/integrations/sailpoint/scim_guard.py @@ -13,11 +13,13 @@ from __future__ import annotations -from typing import Iterable, Optional, Set +from typing import Any, Dict, Iterable, Optional, Set from ..... import api from .....params import KeeperParams +_ALL_PSEUDO_USER = '@all' + class SailPointScimGuard: """Detect SCIM-managed users/nodes and block identity stomps.""" @@ -40,15 +42,21 @@ def _scim_node_ids(params: KeeperParams) -> Set[int]: return nodes @staticmethod - def _node_ancestors(params: KeeperParams, node_id: int) -> Iterable[int]: - by_id = {int(n['node_id']): n for n in params.enterprise.get('nodes') or []} + def _node_parents(params: KeeperParams) -> Dict[int, Optional[int]]: + return { + int(n['node_id']): (int(n['parent_id']) if n.get('parent_id') else None) + for n in params.enterprise.get('nodes') or [] + } + + @staticmethod + def _walk_ancestors(node_id: int, parents: Dict[int, Optional[int]]) -> Iterable[int]: current: Optional[int] = node_id seen = set() while current and current not in seen: yield current seen.add(current) - parent = by_id.get(current, {}).get('parent_id') - current = int(parent) if parent and int(parent) != current else None + parent = parents.get(current) + current = parent if parent != current else None @classmethod def is_scim_managed_node(cls, params: KeeperParams, node_id: Optional[int]) -> bool: @@ -58,35 +66,82 @@ def is_scim_managed_node(cls, params: KeeperParams, node_id: Optional[int]) -> b scim_nodes = cls._scim_node_ids(params) if not scim_nodes: return False - return any(n in scim_nodes for n in cls._node_ancestors(params, int(node_id))) + parents = cls._node_parents(params) + return any(n in scim_nodes for n in cls._walk_ancestors(int(node_id), parents)) + + @staticmethod + def _normalize(value: Any) -> str: + return str(value).strip().lower() if isinstance(value, str) else '' @classmethod - def find_user(cls, params: KeeperParams, email: str): + def _build_user_lookup(cls, params: KeeperParams) -> Dict[str, Any]: + """Mirrors EnterpriseUserCommand's own lookup: id, username, then aliases.""" + lookup: Dict[str, Any] = {} + for user in params.enterprise.get('users') or []: + if 'enterprise_user_id' in user: + lookup[str(user['enterprise_user_id']).strip()] = user + username = cls._normalize(user.get('username')) + if username: + lookup[username] = user + for alias in params.enterprise.get('user_aliases') or []: + username = cls._normalize(alias.get('username')) + if username and username not in lookup: + user_id = str(alias.get('enterprise_user_id')).strip() + if user_id in lookup: + lookup[username] = lookup[user_id] + return lookup + + @classmethod + def _is_all_pseudo_user(cls, identifier: str) -> bool: + return cls._normalize(identifier) == _ALL_PSEUDO_USER + + @classmethod + def find_user(cls, params: KeeperParams, identifier: str) -> Optional[Dict[str, Any]]: cls.ensure_enterprise(params) if not params.enterprise: return None - target = email.strip().lower() - return next( - ( - user for user in params.enterprise.get('users') or [] - if (user.get('username') or '').lower() == target - ), - None, - ) + target = cls._normalize(identifier) + if not target: + return None + return cls._build_user_lookup(params).get(target) @classmethod - def is_scim_managed_user(cls, params: KeeperParams, email: str) -> bool: - user = cls.find_user(params, email) + def is_scim_managed_user(cls, params: KeeperParams, identifier: str) -> bool: + user = cls.find_user(params, identifier) if not user: return False return cls.is_scim_managed_node(params, user.get('node_id')) @classmethod - def identity_change_error(cls, params: KeeperParams, email: str) -> Optional[str]: - if cls.is_scim_managed_user(params, email): + def is_scim_managed_identifier(cls, params: KeeperParams, identifier: str) -> bool: + """Like is_scim_managed_user, but '@all' matches if any user is SCIM-managed.""" + if not cls._is_all_pseudo_user(identifier): + return cls.is_scim_managed_user(params, identifier) + cls.ensure_enterprise(params) + if not params.enterprise: + return False + scim_nodes = cls._scim_node_ids(params) + if not scim_nodes: + return False + parents = cls._node_parents(params) + return any( + any(n in scim_nodes for n in cls._walk_ancestors(int(user['node_id']), parents)) + for user in params.enterprise.get('users') or [] + if user.get('node_id') + ) + + @classmethod + def identity_change_error(cls, params: KeeperParams, identifier: str) -> Optional[str]: + if not cls.is_scim_managed_identifier(params, identifier): + return None + if cls._is_all_pseudo_user(identifier): return ( - f'User {email} is managed by an existing SCIM provider. ' + '@all includes one or more users managed by an existing SCIM provider. ' 'SailPoint may only change folder/record (and admin) entitlements; ' - 'node/team/role identity changes are not allowed.' + 'node/team/role identity changes are not allowed for those users.' ) - return None + return ( + f'User {identifier} is managed by an existing SCIM provider. ' + 'SailPoint may only change folder/record (and admin) entitlements; ' + 'node/team/role identity changes are not allowed.' + ) diff --git a/keepercommander/service/commands/integrations/sailpoint/service.py b/keepercommander/service/commands/integrations/sailpoint/service.py index a1010685b..6d5fd5ad4 100644 --- a/keepercommander/service/commands/integrations/sailpoint/service.py +++ b/keepercommander/service/commands/integrations/sailpoint/service.py @@ -25,14 +25,7 @@ class SailPointService: - """ - Entry point for SailPoint Service Mode. - - Pending entitlements and SailPoint settings live on a dedicated vault - config record (``Commander Service Mode SailPoint Config``). Runtime - resolves that UID from ``SAILPOINT_RECORD`` (compose env) or - ``params.sailpoint_record_uid``. Docker config stays on ``COMMANDER_RECORD``. - """ + """Entry point for SailPoint Service Mode; settings/pending entitlements live on the record pinned by SAILPOINT_RECORD.""" PARAMS_ATTR = PARAMS_ATTR @@ -50,6 +43,16 @@ def record_has_marker(cls, params: KeeperParams, record_uid: str) -> bool: for field in record.custom ) + @classmethod + def _record_has_marker_safe(cls, params: KeeperParams, uid: str) -> bool: + """record_has_marker, but never raises; startup-only -- handle_command/after_command + must call record_has_marker directly so a check failure denies the request instead of silently skipping it.""" + try: + return cls.record_has_marker(params, uid) + except Exception as e: + logger.warning(f'SailPoint: marker check failed for record {uid}: {e}') + return False + @classmethod def record_uid(cls, params: Optional[KeeperParams] = None) -> Optional[str]: if params is not None: @@ -68,22 +71,13 @@ def bind_params(cls, params: KeeperParams, record_uid: Optional[str] = None) -> @classmethod def maybe_enable(cls, params: KeeperParams, args) -> None: - """ - Bind params and sanitize the Service Mode command allowlist when the - SailPoint config record has the integration marker. - - Callers must gate on ``SAILPOINT_RECORD`` before invoking this. - """ + """Bind params and sanitize the allowlist when the config record has the marker; callers must gate on SAILPOINT_RECORD first.""" uid = cls.record_uid(params) - try: - if not cls.record_has_marker(params, uid): - logger.warning( - f'{SAILPOINT_RECORD_ENV}={uid} is set but record is missing ' - f'{SAILPOINT_MARKER_FIELD}; SailPoint mode not enabled' - ) - return - except Exception as e: - logger.warning(f'SailPoint marker check failed; mode not enabled: {e}') + if not cls._record_has_marker_safe(params, uid): + logger.warning( + f'{SAILPOINT_RECORD_ENV}={uid} is set but record is missing ' + f'{SAILPOINT_MARKER_FIELD}; SailPoint mode not enabled' + ) return cls.bind_params(params, uid) @@ -98,25 +92,17 @@ def maybe_enable(cls, params: KeeperParams, args) -> None: @classmethod def start_background_services(cls) -> None: - """ - Start the entitlement poller when SailPoint is enabled. - - Callers must gate on ``SAILPOINT_RECORD`` before invoking this. - """ + """Start the entitlement poller; callers must gate on SAILPOINT_RECORD before invoking this.""" from ....core.globals import get_current_params params = get_current_params() if not params: logger.warning('SailPoint poller not started: Keeper params not loaded') return uid = cls.record_uid(params) - try: - if not cls.record_has_marker(params, uid): - logger.warning( - f'SailPoint poller not started: record {uid} missing {SAILPOINT_MARKER_FIELD}' - ) - return - except Exception as e: - logger.warning(f'SailPoint poller not started: marker check failed: {e}') + if not cls._record_has_marker_safe(params, uid): + logger.warning( + f'SailPoint poller not started: record {uid} missing {SAILPOINT_MARKER_FIELD}' + ) return cls.bind_params(params, uid) try: @@ -129,15 +115,12 @@ def start_background_services(cls) -> None: def handle_command( cls, params: KeeperParams, command: str ) -> Tuple[str, Optional[Tuple[Any, int]]]: - """ - Prepare a SailPoint Service Mode command. - - Returns ``(command_to_run, short_circuit)``. Callers must gate on - ``SAILPOINT_RECORD`` before invoking this. - """ + """Prepare a SailPoint command, returning (command_to_run, short_circuit); callers must gate on SAILPOINT_RECORD first.""" cls.bind_params(params) uid = cls.record_uid(params) + # Not swallowed here -- a check failure must deny the request, not silently skip gating. if not cls.record_has_marker(params, uid): + logger.debug(f'SailPoint: record {uid} has no active marker; skipping all command gating for this request') return command, None return SailPointCommandHook(uid).before_command(params, command) @@ -146,6 +129,8 @@ def after_command(cls, params: KeeperParams, command: str, success: bool = True) """Callers must gate on ``SAILPOINT_RECORD`` before invoking this.""" cls.bind_params(params) uid = cls.record_uid(params) + # Not swallowed here either -- callers turn a raised exception into an explicit 500. if not cls.record_has_marker(params, uid): + logger.debug(f'SailPoint: record {uid} has no active marker; skipping pending-entitlement queue for this request') return SailPointCommandHook(uid).after_command(params, command, success) diff --git a/keepercommander/service/commands/integrations/sailpoint_app_setup.py b/keepercommander/service/commands/integrations/sailpoint_app_setup.py index 875eeee1d..71c27c6e5 100644 --- a/keepercommander/service/commands/integrations/sailpoint_app_setup.py +++ b/keepercommander/service/commands/integrations/sailpoint_app_setup.py @@ -33,6 +33,7 @@ MIN_POLL_INTERVAL_SECONDS, PENDING_ENTITLEMENTS_FIELD, POLL_INTERVAL_FIELD, + SAILPOINT_BANNED_COMMANDS, SAILPOINT_MARKER_FIELD, SAILPOINT_RECORD_ENV, TRANSFER_TARGET_EMAIL_FIELD, @@ -60,6 +61,9 @@ def get_record_env_key(self) -> str: def get_service_commands(self) -> str: return SailPointCommandPolicy.default_allowlist() + def get_banned_commands(self) -> tuple: + return tuple(SAILPOINT_BANNED_COMMANDS) + def collect_integration_config(self, params, transfer_target_default: str = ''): print(f"\n{bcolors.BOLD}SHARE ENTITLEMENTS:{bcolors.ENDC}") print(f" Control which share entitlements SailPoint may manage via Service Mode") @@ -186,7 +190,7 @@ def _update_docker_compose(self, setup_result, service_config, record_uid, confi try: cfg = asdict(service_config) - cfg['commands'] = SailPointCommandPolicy.sanitize(cfg.get('commands') or '') + cfg['commands'] = self.sanitize_service_commands(cfg.get('commands') or '') builder = DockerComposeBuilder( setup_result, cfg, diff --git a/keepercommander/service/commands/terraform_app_setup.py b/keepercommander/service/commands/terraform_app_setup.py index b802f0965..811f3e5da 100644 --- a/keepercommander/service/commands/terraform_app_setup.py +++ b/keepercommander/service/commands/terraform_app_setup.py @@ -26,6 +26,7 @@ ) from ..decorators.min_commander_version import TERRAFORM_DOCKER_ENV from ..util.exceptions import ValidationError +from .integrations.command_policy import sanitize_commands from .service_docker_setup import ServiceDockerSetupCommand @@ -47,7 +48,7 @@ class TerraformSetupConstants: 'share-folder', 'rmdir', 'rndir', 'mkdir', 'epm', 'scim', 'mv', 'pam', 'secrets-manager', 'ln', 'share-record', 'nsf-mkdir', 'nsf-get', 'nsf-rmdir', 'nsf-record-add', 'nsf-record-update', - 'nsf-rm', 'nsf-rndir', 'nsf-share-folder', 'nsf-share-record', 'nsf-ln', + 'nsf-rm', 'nsf-rndir', 'nsf-share-folder', 'nsf-share-record', 'nsf-ln', 'nsf-move', ) SERVICE_COMMANDS = ','.join(SERVICE_COMMANDS_LIST) @@ -113,7 +114,7 @@ def execute(self, params, **kwargs): def _validate_terraform_commands(self, params) -> str: try: - return RuntimeServiceConfig().validate_command_list( + validated = RuntimeServiceConfig().validate_command_list( TerraformSetupConstants.SERVICE_COMMANDS, params ) except ValidationError as e: @@ -121,6 +122,7 @@ def _validate_terraform_commands(self, params) -> str: self.get_parser().prog, f'Terraform command allowlist validation failed: {e}', ) + return sanitize_commands(validated, TerraformSetupConstants.SERVICE_COMMANDS_LIST) def _get_commands_config(self, params) -> str: return self._validate_terraform_commands(params) @@ -134,7 +136,7 @@ def generate_docker_compose_yaml(self, setup_result: SetupResult, config: Docker asdict(config), commander_service_name=TerraformSetupConstants.COMMANDER_SERVICE_NAME, commander_container_name=TerraformSetupConstants.COMMANDER_CONTAINER_NAME, - commander_environment={TERRAFORM_DOCKER_ENV: '1'}, + commander_environment={TERRAFORM_DOCKER_ENV: setup_result.record_uid}, ) return builder.build() diff --git a/keepercommander/service/config/cloudflare_config.py b/keepercommander/service/config/cloudflare_config.py index ea9fda01c..a6a80d871 100644 --- a/keepercommander/service/config/cloudflare_config.py +++ b/keepercommander/service/config/cloudflare_config.py @@ -16,7 +16,7 @@ from ..decorators.logging import logger, debug_decorator from .service_config import ServiceConfig -from ..util.tunneling import generate_cloudflare_url +from ..util.tunneling import generate_cloudflare_url, get_tunnel_log_file from ..util.exceptions import ValidationError class CloudflareConfigurator: @@ -86,8 +86,7 @@ def _check_cloudflare_tunnel_health(max_wait_seconds=None): @staticmethod def _get_cloudflare_log_path() -> str: """Get the path to the Cloudflare tunnel log file.""" - service_core_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "core") - return os.path.join(service_core_dir, "logs", "cloudflare_tunnel_subprocess.log") + return get_tunnel_log_file("cloudflare_tunnel_subprocess.log") @staticmethod def _analyze_tunnel_log(log_file: str) -> Tuple[Optional[bool], str]: diff --git a/keepercommander/service/core/service_app.py b/keepercommander/service/core/service_app.py index 051df174a..3d182a5ef 100644 --- a/keepercommander/service/core/service_app.py +++ b/keepercommander/service/core/service_app.py @@ -9,17 +9,33 @@ # Contact: ops@keepersecurity.com # -from ...service.app import create_app -from ...service.config.service_config import ServiceConfig -from ...service.core.service_manager import ServiceManager +import sys -flask_app = create_app() +# Argv flag (not an env var, which could leak into subprocesses) signaling a frozen background service. +SERVICE_MODE_FLAG = '--internal-run-service' -if __name__ == '__main__': +def run_background_service(): + """ + Run the Flask service in background mode. + This function is called both when running as a module (-m) and + when the frozen executable is invoked with SERVICE_MODE_FLAG. + """ + # PyInstaller's bootloader doesn't reliably honor PYTHONUNBUFFERED, so force it here too. + for stream in (sys.stdout, sys.stderr): + if stream is not None and hasattr(stream, 'reconfigure'): + stream.reconfigure(line_buffering=True) + + from ...service.app import create_app + from ...service.config.service_config import ServiceConfig + from ...service.core.service_manager import ServiceManager + + flask_app = create_app() + service_config = ServiceConfig() config_data = service_config.load_config() + # Pre-load Keeper parameters for background mode try: from ...service.core.globals import ensure_params_loaded print("Pre-loading Keeper parameters for background mode...") @@ -29,10 +45,10 @@ print(f"Warning: Failed to pre-load parameters during startup: {e}") print("Parameters will be loaded on first API call if needed") - ssl_context = None - - if not (port := config_data.get("port")): + port = config_data.get("port") + if not port: print("Error: Service configuration is incomplete. Please configure the service port in service_config") + sys.exit(1) ssl_context = ServiceManager.get_ssl_context(config_data) @@ -42,3 +58,7 @@ ssl_context=ssl_context ) + +if __name__ == '__main__': + run_background_service() + diff --git a/keepercommander/service/core/service_manager.py b/keepercommander/service/core/service_manager.py index 0d910237c..d65d0c449 100644 --- a/keepercommander/service/core/service_manager.py +++ b/keepercommander/service/core/service_manager.py @@ -17,6 +17,7 @@ from ... import utils from ...service.config.service_config import ServiceConfig from ..decorators.logging import logger, debug_decorator +from ..util.process_util import spawn_detached_process, CREATE_NO_WINDOW from .process_info import ProcessInfo from .terminal_handler import TerminalHandler from .signal_handler import SignalHandler @@ -92,7 +93,12 @@ def start_service(cls) -> None: else: print(f"Commander Service starting on \033[1m{protocol}://localhost:{port}/api/v1/executecommand\033[0m") - ngrok_pid = NgrokConfigurator.configure_ngrok(config_data, service_config) + try: + ngrok_pid = NgrokConfigurator.configure_ngrok(config_data, service_config) + except Exception as e: + ProcessInfo.clear() + logger.error(f"\n{str(e)}") + return cloudflare_pid = None try: @@ -110,7 +116,7 @@ def start_service(cls) -> None: ProcessInfo.clear() - logger.info(f"\n{str(e)}") + logger.error(f"\n{str(e)}") return # Custom logging filter to replace SSL handshake errors with user-friendly message @@ -130,43 +136,41 @@ def filter(self, record): werkzeug_logger.addFilter(SSLHandshakeFilter()) if config_data.get("run_mode") == "background": - - base_dir = os.path.dirname(os.path.abspath(__file__)) - service_module = "keepercommander.service.core.service_app" # Use module path instead of file path - python_executable = sys.executable - - # Create logs directory for subprocess output - log_dir = os.path.join(base_dir, "logs") + + # Detect if running as PyInstaller executable + is_frozen = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') + + # Create logs directory for subprocess output in user-writable location + log_dir = os.path.join(utils.get_default_path(), "service_logs") os.makedirs(log_dir, exist_ok=True) log_file = os.path.join(log_dir, "service_subprocess.log") try: - if sys.platform == "win32": - subprocess.DETACHED_PROCESS = 0x00000008 - with open(log_file, 'w') as log_f: - cls = subprocess.Popen( - [python_executable, '-m', service_module], - creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP, - stdout=log_f, - stderr=subprocess.STDOUT, # Combine stderr with stdout - cwd=os.getcwd(), # Use current working directory to access config files - env=os.environ.copy() # Inherit environment variables - ) + python_executable = sys.executable + + # Set up environment for subprocess + subprocess_env = os.environ.copy() + # Redirected (non-TTY) stdout defaults to full buffering, delaying log output. + subprocess_env['PYTHONUNBUFFERED'] = '1' + + if is_frozen: + # -m doesn't work for a frozen exe, so pass an explicit internal flag instead. + from .service_app import SERVICE_MODE_FLAG + cmd = [python_executable, SERVICE_MODE_FLAG] else: - # For macOS and Linux - improved subprocess handling - with open(log_file, 'w') as log_f: - cls = subprocess.Popen( - [python_executable, '-m', service_module], - stdout=log_f, - stderr=subprocess.STDOUT, # Combine stderr with stdout - preexec_fn=os.setpgrp, - cwd=os.getcwd(), # Use current working directory to access config files - env=os.environ.copy() # Inherit environment variables - ) - + # Running as Python script - use -m flag + cmd = [python_executable, '-m', 'keepercommander.service.core.service_app'] + + # append=True to preserve history across restarts (tunnel logs truncate instead). + process = spawn_detached_process( + cmd, log_file, cwd=os.getcwd(), env=subprocess_env, append=True + ) + # Command output can include vault data - don't leave it world-readable. + utils.set_file_permissions(log_file) + logger.debug(f"Service subprocess logs available at: {log_file}") - print(f"Commander Service started with PID: {cls.pid}") - ProcessInfo.save(cls.pid, is_running, ngrok_pid) + print(f"Commander Service started with PID: {process.pid}") + ProcessInfo.save(process.pid, is_running, ngrok_pid, cloudflare_pid) except Exception as e: logger.error(f"Failed to start service subprocess: {e}") @@ -257,7 +261,7 @@ def foreground_signal_handler(signum, frame): cls._flask_app = create_app() cls._is_running = True - ProcessInfo.save(os.getpid(), is_running, ngrok_pid) + ProcessInfo.save(os.getpid(), is_running, ngrok_pid, cloudflare_pid) ssl_context = ServiceManager.get_ssl_context(config_data) try: @@ -268,9 +272,6 @@ def foreground_signal_handler(signum, frame): ) finally: cleanup_cloudflare_on_foreground_exit() - - # Save the process ID for future reference - ProcessInfo.save(cls.pid, is_running, ngrok_pid, cloudflare_pid) except FileNotFoundError: logging.info("Error: Service configuration file not found. Please use 'service-create' command to create a service_config file.") @@ -459,7 +460,13 @@ def kill_process_by_pid(pid: int): try: if sys.platform.startswith("win"): # Windows logger.debug(f"Using Windows taskkill for PID {pid}") - subprocess.run(["taskkill", "/PID", str(pid), "/F"], check=True) + subprocess.run( + ["taskkill", "/PID", str(pid), "/F"], + check=True, + creationflags=CREATE_NO_WINDOW, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) return True else: # Linux & macOS try: diff --git a/keepercommander/service/decorators/api_logging.py b/keepercommander/service/decorators/api_logging.py index d06ae84f2..74ee593d2 100644 --- a/keepercommander/service/decorators/api_logging.py +++ b/keepercommander/service/decorators/api_logging.py @@ -14,11 +14,8 @@ from typing import Callable, Any from flask import request import time -from .logging import logger, sanitize_command_fields, SENSITIVE_FIELD_TYPES - -# Legacy generic keys (kept for JSON payloads that aren't shaped like Keeper -# record fields, e.g. arbitrary nested config blobs). -_SENSITIVE_DICT_KEYS = frozenset({'password', 'login', 'secret', 'token', 'key'}) | SENSITIVE_FIELD_TYPES +from .logging import logger, sanitize_command_fields +from ...sanitization import sanitize_nested_data class SSLHandshakeFilter(logging.Filter): @@ -44,48 +41,9 @@ def sanitize_password_in_command(data): # Sanitize filedata if present if 'filedata' in sanitized: - sanitized['filedata'] = _sanitize_nested_data(sanitized['filedata']) - - return sanitized - -def _mask_field_value(value): - """Mask a Keeper record field's `value`, preserving its container shape.""" - if isinstance(value, list): - return ['***' for _ in value] - if isinstance(value, dict): - return {k: '***' for k in value} - return '***' - + sanitized['filedata'] = sanitize_nested_data(sanitized['filedata']) -def _sanitize_nested_data(data): - """Recursively sanitize nested data structures""" - if isinstance(data, dict): - field_type = data.get('type') - if isinstance(field_type, str) and field_type.lower() in SENSITIVE_FIELD_TYPES and 'value' in data: - sanitized = dict(data) - sanitized['value'] = _mask_field_value(data['value']) - return sanitized - - sanitized = {} - for key, value in data.items(): - # Sanitize sensitive field names - if key.lower() in _SENSITIVE_DICT_KEYS: - if isinstance(value, str) and len(value) > 0: - sanitized[key] = '*' * min(len(value), 15) - else: - sanitized[key] = '***' - else: - sanitized[key] = _sanitize_nested_data(value) - return sanitized - elif isinstance(data, list): - return [_sanitize_nested_data(item) for item in data] - elif isinstance(data, str): - # Sanitize email addresses in string values to protect PII - import re - sanitized_str = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '***@***.***', data) - return sanitized_str - else: - return data + return sanitized def _get_sanitized_request_data(): """Extract and sanitize request data for logging (only for JSON POST requests)""" @@ -94,6 +52,9 @@ def _get_sanitized_request_data(): try: json_data = request.get_json(silent=True) sanitized_data = sanitize_password_in_command(json_data) + # Additional sanitization for nested structures that might contain sensitive data + if sanitized_data and isinstance(sanitized_data, dict): + sanitized_data = sanitize_nested_data(sanitized_data) except Exception: sanitized_data = None return f"data={sanitized_data}" if sanitized_data else "no-data" diff --git a/keepercommander/service/decorators/logging.py b/keepercommander/service/decorators/logging.py index 7542aeb0f..44b4f8825 100644 --- a/keepercommander/service/decorators/logging.py +++ b/keepercommander/service/decorators/logging.py @@ -17,14 +17,7 @@ import shlex from enum import Enum from ... import utils - -# Values that must never reach the logs when set via record-add/record-update/ -# nsf-record-* CLI args. -SENSITIVE_FIELD_TYPES = frozenset({ - 'password', 'login', 'secret', 'onetimecode', 'pincode', 'keypair', - 'privatekey', 'passphrase', 'paymentcard', 'bankaccount', - 'securityquestion', 'passkey', -}) +from ...sanitization import SENSITIVE_FIELD_TYPES class LogLevel(Enum): ERROR = logging.ERROR @@ -68,7 +61,7 @@ def _creat_logging_config(self): return default_config["logging"] def _load_config(self): - config_path = utils.get_default_path() / "logging_config.yaml"; + config_path = utils.get_default_path() / "logging_config.yaml" # config_path = os.getenv("LOGGING_CONFIG_PATH", "logging_config.yaml") if os.path.exists(config_path): @@ -150,24 +143,24 @@ def sanitize_debug_data(data: str) -> str: """Sanitize sensitive data from debug output.""" if not data: return data - + sanitized = data - + # Sanitize common password patterns patterns = [ (r'"password"\s*:\s*"[^"]*"', '"password": "***"'), - (r'"login"\s*:\s*"[^"]*"', '"login": "***"'), (r'"secret"\s*:\s*"[^"]*"', '"secret": "***"'), (r'"token"\s*:\s*"[^"]*"', '"token": "***"'), (r'"key"\s*:\s*"[^"]*"', '"key": "***"'), + (r'"licenseNumber"\s*:\s*"[^"]*"', '"licenseNumber": "***"'), + (r'"encryptedNote"\s*:\s*"[^"]*"', '"encryptedNote": "***"'), + (r'"note"\s*:\s*"[^"]*"', '"note": "***"'), + # Bare field formats (e.g., password=value, secret=value) (r'\bpassword=[^\s]*', 'password=***'), - (r'\blogin=[^\s]*', 'login=***'), - # oneTimeCode=otpauth://totp/...?secret=... — mask the whole value, TOTP seed included (r'\boneTimeCode=[^\s]*', 'oneTimeCode=***'), (r'\bsecret=[^\s]*', 'secret=***'), - # Other sensitive record field types (see SENSITIVE_FIELD_TYPES) that can - # appear as bare CLI args on record-add/record-update/nsf-* commands. (r'\bpinCode=[^\s]*', 'pinCode=***'), + (r'\bnote=[^\s]*', 'note=***'), (r'\bkeyPair=[^\s]*', 'keyPair=***'), (r'\bprivateKey=[^\s]*', 'privateKey=***'), (r'\bpassphrase=[^\s]*', 'passphrase=***'), @@ -175,10 +168,39 @@ def sanitize_debug_data(data: str) -> str: (r'\bbankAccount=[^\s]*', 'bankAccount=***'), (r'\bsecurityQuestion=[^\s]*', 'securityQuestion=***'), (r'\bpasskey=[^\s]*', 'passkey=***'), - # Sanitize email addresses in logs to protect PII - (r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '***@***.***'), + (r'\blicenseNumber=[^\s]*', 'licenseNumber=***'), + (r'\bencryptedNote=[^\s]*', 'encryptedNote=***'), + # Command options (--password, etc) + (r'--password=[^\s]*', '--password=***'), + (r'--password\s+[^\s]+', '--password ***'), + # Prefixed field formats (f.fieldName=value, c.fieldName=value) + (r'\bf\.bankAccount\.accountNumber=[^\s]*', 'f.bankAccount.accountNumber=***'), + (r'\bf\.bankAccount\.routingNumber=[^\s]*', 'f.bankAccount.routingNumber=***'), + (r'\bc\.bankAccount\.accountNumber=[^\s]*', 'c.bankAccount.accountNumber=***'), + (r'\bc\.bankAccount\.routingNumber=[^\s]*', 'c.bankAccount.routingNumber=***'), + (r'\bf\.paymentCard\.cardNumber=[^\s]*', 'f.paymentCard.cardNumber=***'), + (r'\bf\.paymentCard\.cardSecurityCode=[^\s]*', 'f.paymentCard.cardSecurityCode=***'), + (r'\bc\.paymentCard\.cardNumber=[^\s]*', 'c.paymentCard.cardNumber=***'), + (r'\bc\.paymentCard\.cardSecurityCode=[^\s]*', 'c.paymentCard.cardSecurityCode=***'), + (r'\bf\.keyPair\.privateKey=[^\s]*', 'f.keyPair.privateKey=***'), + (r'\bf\.keyPair\.publicKey=[^\s]*', 'f.keyPair.publicKey=***'), + (r'\bc\.keyPair\.privateKey=[^\s]*', 'c.keyPair.privateKey=***'), + (r'\bc\.keyPair\.publicKey=[^\s]*', 'c.keyPair.publicKey=***'), + (r'\bf\.licenseNumber=[^\s]*', 'f.licenseNumber=***'), + (r'\bc\.licenseNumber=[^\s]*', 'c.licenseNumber=***'), + (r'\bf\.encryptedNote=[^\s]*', 'f.encryptedNote=***'), + (r'\bc\.encryptedNote=[^\s]*', 'c.encryptedNote=***'), + # Custom field labels with sensitive types (f.password.Label=, c.secret.Label=, etc.) + (r'\b[fc]\.password\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.secret\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.keypair\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.privatekey\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.bankaccount\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.paymentcard\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.licensenumber\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.encryptednote\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), ] - + for pattern, replacement in patterns: sanitized = re.sub(pattern, replacement, sanitized, flags=re.IGNORECASE) diff --git a/keepercommander/service/decorators/min_commander_version.py b/keepercommander/service/decorators/min_commander_version.py index 4d484caf3..64f336b78 100644 --- a/keepercommander/service/decorators/min_commander_version.py +++ b/keepercommander/service/decorators/min_commander_version.py @@ -23,8 +23,11 @@ # Hyphenated only: Werkzeug/WSGI silently drops headers that contain underscores. MIN_COMMANDER_VERSION_HEADER = 'Min-Commander-Version' -# Set on terraform-app-setup compose; not a secret — instance identity only. -TERRAFORM_DOCKER_ENV = 'KEEPER_TERRAFORM' +# Set on terraform-app-setup compose to the Terraform config record's UID. +TERRAFORM_DOCKER_ENV = 'TERRAFORM_RECORD' +# Pre-rename value (was '1', not a UID). Recognized so containers upgraded without re-running +# terraform-app-setup don't silently lose enforcement; drop after a migration period. +TERRAFORM_DOCKER_ENV_LEGACY = 'KEEPER_TERRAFORM' def _parse_version(version_str: str) -> Optional[Version]: @@ -41,8 +44,11 @@ def _parse_version(version_str: str) -> Optional[Version]: def _is_terraform_docker() -> bool: - """True when this process was started from terraform-app-setup compose.""" - return bool((os.environ.get(TERRAFORM_DOCKER_ENV) or '').strip()) + """True when this process was started from terraform-app-setup compose (new or pre-rename env var).""" + return bool( + (os.environ.get(TERRAFORM_DOCKER_ENV) or '').strip() + or (os.environ.get(TERRAFORM_DOCKER_ENV_LEGACY) or '').strip() + ) def _read_min_commander_version_header() -> Optional[str]: diff --git a/keepercommander/service/util/command_util.py b/keepercommander/service/util/command_util.py index 910c2c42f..ae2e2dade 100644 --- a/keepercommander/service/util/command_util.py +++ b/keepercommander/service/util/command_util.py @@ -25,6 +25,13 @@ is_throttle_error, throttle_error_response, ) +from .protected_records import ( + get_protected_folder_uids, + get_protected_record_uids, + hide_from_folder_cache, + hide_from_record_cache, + resolve_sync_down_exempt_uid, +) from .verified_command import Verifycommand from ..core.globals import get_current_params from ..decorators.logging import logger, debug_decorator, sanitize_debug_data, sanitize_command_fields @@ -172,16 +179,18 @@ def execute(cls, command: str, temp_files: Optional[list] = None) -> Tuple[Any, # Mode will treat as safe, not the whole shared OS temp root. request_temp_dir = os.path.dirname(temp_files[0]) if temp_files else None + def blocked(error): + logger.warning( + f"Service Mode blocked command '{command_tokens[0] if command_tokens else ''}': {error}" + ) + return {"status": "error", "error": error}, 403 + # Same tokens the CLI will run — do not use raw HTTP split(" ") service_mode_error = Verifycommand.validate_service_mode_restrictions( command_tokens, request_temp_dir ) if service_mode_error: - logger.warning( - f"Service Mode blocked command '{command_tokens[0] if command_tokens else ''}': " - f"{service_mode_error}" - ) - return {"status": "error", "error": service_mode_error}, 403 + return blocked(service_mode_error) force_error = Verifycommand.validate_enterprise_user_add_role_force( command_tokens, params @@ -189,16 +198,46 @@ def execute(cls, command: str, temp_files: Optional[list] = None) -> Tuple[Any, if force_error: return {"status": "error", "error": force_error}, 400 - sailpoint_enabled = bool((os.environ.get('SAILPOINT_RECORD') or '').strip()) - if sailpoint_enabled: - from ..commands.integrations.sailpoint.service import SailPointService - command, sailpoint_response = SailPointService.handle_command(params, command) - if sailpoint_response is not None: - response, status_code = sailpoint_response - response = CommandExecutor.encrypt_response(response) - return response, status_code + # Checked for every command (not a curated list) so no current or future + # command can be missed as a way to reference these records. + protected_uids = get_protected_record_uids(params) + + # {slack,gchat}-app-setup --sync-down needs its own config record reachable. + sync_down_exempt_uid = resolve_sync_down_exempt_uid(command_tokens) + if sync_down_exempt_uid is not None: + protected_uids = { + uid: title for uid, title in protected_uids.items() if uid != sync_down_exempt_uid + } + + # Derived from the record set so the exemption above reaches the exempted integration's own folder too. + protected_folder_uids = get_protected_folder_uids(params, protected_uids) + + protected_command_error = Verifycommand.validate_service_mode_protected_record_command( + command_tokens, + {**protected_uids, **{uid: '' for uid in protected_folder_uids}}, + ) + if protected_command_error: + return blocked(protected_command_error) + + sailpoint_uid = (os.environ.get('SAILPOINT_RECORD') or '').strip() + sailpoint_enabled = bool(sailpoint_uid) + + # Only SailPoint's own record is exempt from handle_command's guard; every other + # protected record (and the folder cache, which has no exemption) stays hidden throughout. + handle_command_uids = {uid: title for uid, title in protected_uids.items() if uid != sailpoint_uid} + + with hide_from_folder_cache(params, protected_folder_uids), \ + hide_from_record_cache(params, handle_command_uids): + if sailpoint_enabled: + from ..commands.integrations.sailpoint.service import SailPointService + command, sailpoint_response = SailPointService.handle_command(params, command) + if sailpoint_response is not None: + response, status_code = sailpoint_response + response = CommandExecutor.encrypt_response(response) + return response, status_code - return_value, printed_output, log_output = CommandExecutor.capture_output_and_logs(params, command) + with hide_from_record_cache(params, protected_uids): + return_value, printed_output, log_output = CommandExecutor.capture_output_and_logs(params, command) response = return_value if return_value else printed_output # Debug logging with sanitization @@ -214,7 +253,10 @@ def execute(cls, command: str, temp_files: Optional[list] = None) -> Tuple[Any, if status_code == 200 and sailpoint_enabled: try: - SailPointService.after_command(params, command, success=True) + # Same rule as the pre-dispatch phase: every protected record except + # SailPoint's own stays hidden here too. + with hide_from_record_cache(params, handle_command_uids): + SailPointService.after_command(params, command, success=True) except Exception as e: logger.error(f'SailPoint post-process failed: {sanitize_debug_data(str(e))}') err = { diff --git a/keepercommander/service/util/process_util.py b/keepercommander/service/util/process_util.py new file mode 100644 index 000000000..d02fffac4 --- /dev/null +++ b/keepercommander/service/util/process_util.py @@ -0,0 +1,46 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' ', + 'TERRAFORM_RECORD': '', + 'SLACK_RECORD': '', + 'TEAMS_RECORD': '', + 'GCHAT_RECORD': '', + 'SAILPOINT_RECORD': '', +} + +# uid-keyed caches resolve_single_record/load_pam_record fall back to when a UID isn't in record_cache. +_GUARDED_CACHE_ATTRS = ('record_cache', 'nested_share_records', 'nested_share_record_data') + +# Raw + derived folder caches every folder-resolving command reads through, +# directly or via subfolder.try_resolve_path/get_folder_uids. +_GUARDED_FOLDER_CACHE_ATTRS = ('folder_cache', 'shared_folder_cache', 'subfolder_cache') + +# Commander's own session (config.json) and Service Mode's own runtime (service_config.json) +# config files -- always attached under these exact, hardcoded names, never user-choosable. +_RESERVED_ATTACHMENT_NAMES = frozenset({'config.json', 'service_config.json'}) + + +def _attachment_file_uids(record) -> list: + """UIDs of every file attachment on record -- legacy PasswordRecord.attachments' ids, or a + typed record's fileRef entries (each its own separate FileRecord, loadable/gettable by that UID).""" + from ... import vault + + if isinstance(record, vault.PasswordRecord): + return [atta.id for atta in (record.attachments or []) if atta.id] + if isinstance(record, vault.TypedRecord): + typed_field = record.get_typed_field('fileRef') + if typed_field and isinstance(typed_field.value, list): + return [uid for uid in typed_field.value if isinstance(uid, str)] + return [] + + +def _has_reserved_legacy_attachment(record) -> bool: + """True if a PasswordRecord's own .attachments (no extra load -- filenames live on the attachment + object itself) include one literally named config.json/service_config.json.""" + from ... import vault + + return isinstance(record, vault.PasswordRecord) and any( + (atta.title or atta.name or '').lower() in _RESERVED_ATTACHMENT_NAMES + for atta in (record.attachments or []) + ) + + +def _protected_titles() -> Tuple[str, ...]: + """The literal titles of Service Mode's own config records; imported lazily to avoid a circular import through verified_command.""" + from ..config.file_handler import SERVICE_CONFIG_RECORD_TITLES + from ..commands.terraform_app_setup import TerraformSetupConstants + from ..commands.integrations.slack_app_setup import SlackAppSetupCommand + from ..commands.integrations.teams_app_setup import TeamsAppSetupCommand + from ..commands.integrations.sailpoint_app_setup import SailPointAppSetupCommand + from ..docker.models import DockerSetupConstants, GChatConstants + return ( + *SERVICE_CONFIG_RECORD_TITLES, + DockerSetupConstants.DEFAULT_RECORD_NAME, + TerraformSetupConstants.DEFAULT_RECORD_NAME, + GChatConstants.DEFAULT_RECORD_NAME, + SlackAppSetupCommand().get_default_record_name(), + TeamsAppSetupCommand().get_default_record_name(), + SailPointAppSetupCommand().get_default_record_name(), + ) + + +def get_protected_record_title_set() -> FrozenSet[str]: + """Lower-cased titles of Service Mode's own config records, for literal matching.""" + return frozenset(t.lower() for t in _protected_titles()) + + +def get_protected_record_uids(params) -> Dict[str, str]: + """Resolve current UIDs of Service Mode's own config records ({uid: title}), matching by title plus each integration's pinned UID env var (_PINNED_RECORD_UID_ENVS); not cached, since a stale result on this security check is worse than the cost of a full-vault scan.""" + from ..commands.integrations.approvals_setup import is_valid_keeper_uid + from ..decorators.logging import logger + + found: Dict[str, str] = {} + for env_name, label in _PINNED_RECORD_UID_ENVS.items(): + uid = (os.environ.get(env_name) or '').strip() + if not uid: + continue + if not is_valid_keeper_uid(uid): + logger.warning(f'protected_records: {env_name} is set but not a valid record UID; falling back to title matching for it') + continue + found[uid] = label + + if params is None or not isinstance(getattr(params, 'record_cache', None), dict) or not params.record_cache: + return found + + from ... import vault + + protected_titles = get_protected_record_title_set() + # One load per record_cache entry, no more -- a FileRecord attachment target is itself an + # entry in this same cache, so its name is picked up by this same pass rather than a second, + # per-attachment load + reserved_file_uids: Set[str] = set() + pending_attachments: Dict[str, list] = {} + for uid in params.record_cache: + try: + record = vault.KeeperRecord.load(params, uid) + except Exception as e: + logger.debug(f'protected_records: could not load record {uid} ({type(e).__name__}); skipping') + continue + if not record: + continue + + if isinstance(record, vault.FileRecord): + if (record.title or record.name or '').lower() in _RESERVED_ATTACHMENT_NAMES: + reserved_file_uids.add(uid) + continue + + if record.title.lower() in protected_titles: + found[uid] = record.title + elif _has_reserved_legacy_attachment(record): + found[uid] = '' + + file_uids = _attachment_file_uids(record) + if file_uids: + pending_attachments[uid] = file_uids + + for parent_uid, file_uids in pending_attachments.items(): + if parent_uid not in found and any(file_uid in reserved_file_uids for file_uid in file_uids): + found[parent_uid] = '' + if parent_uid in found: + for file_uid in file_uids: + found.setdefault(file_uid, '') + + return found + + +def get_protected_folder_uids(params, protected_record_uids: Dict[str, str]) -> Set[str]: + """Folders directly containing an already-protected record, via subfolder_record_cache (folder_uid -> set of record UIDs) -- derived from record protection rather than a separate per-integration title list, so it stays correct even if a folder is renamed.""" + subfolder_record_cache = getattr(params, 'subfolder_record_cache', None) + if params is None or not protected_record_uids or not isinstance(subfolder_record_cache, dict): + return set() + + record_uids = protected_record_uids.keys() + return { + folder_uid for folder_uid, uids in subfolder_record_cache.items() + if folder_uid and isinstance(uids, (set, frozenset)) and uids & record_uids + } + + +def _sync_down_exempt_commands() -> Dict[str, str]: + """{command name: pinned-UID env var}, derived from each integration's own class instead of duplicated literals.""" + from ..commands.integrations.gchat_app_setup import GChatAppSetupCommand + from ..commands.integrations.slack_app_setup import SlackAppSetupCommand + return {cmd.get_command_name(): cmd.get_record_env_key() for cmd in (SlackAppSetupCommand(), GChatAppSetupCommand())} + + +def resolve_sync_down_exempt_uid(command_tokens) -> Optional[str]: + """For '{slack,gchat}-app-setup ... --sync-down ...', the one UID this dispatch may bypass Layers A/B for -- always this integration's own pinned-env UID, never derived from what the admin passes (e.g. -r/--integration-record), so a different integration's protected record can never be reached this way.""" + if not command_tokens: + return None + + env_name = _sync_down_exempt_commands().get(command_tokens[0].lower()) + if not env_name: + return None + + if '--sync-down' not in command_tokens[1:]: + return None + + return (os.environ.get(env_name) or '').strip() or None + + +class _GuardedRecordCache(UserDict): + """A uid-keyed cache view that can never hold the given protected UIDs; UserDict (not dict) so every mutation reliably routes through __setitem__, even C-level ones like setdefault/|=.""" + + def __init__(self, source, protected_uids: Iterable[str]): + self._protected_uids = frozenset(protected_uids) + super().__init__({k: v for k, v in source.items() if k not in self._protected_uids}) + + def __setitem__(self, key, value): + if key in self._protected_uids: + return + super().__setitem__(key, value) + + +@contextlib.contextmanager +def hide_from_record_cache(params, protected_uids: Dict[str, str]): + """For the with-block, guards record_cache/nested_share_records/nested_share_record_data against reintroduction (not just a one-time pop) and strips protected UIDs from subfolder_record_cache, restoring everything on exit; relies on Service Mode commands running one at a time (same assumption capture_output_and_logs already makes) and may leave record_cache stale until the next sync if one lands mid-command.""" + if params is None or not protected_uids: + yield + return + + protected_uid_set = frozenset(protected_uids) + + original_caches = {} + saved_entries = {} + for attr in _GUARDED_CACHE_ATTRS: + source = getattr(params, attr, None) + if not isinstance(source, MutableMapping): + continue + original_caches[attr] = source + saved_entries[attr] = {uid: source[uid] for uid in protected_uid_set if uid in source} + setattr(params, attr, _GuardedRecordCache(source, protected_uid_set)) + + subfolder_cache = getattr(params, 'subfolder_record_cache', None) + removed_from_folders: Dict[str, set] = {} + if isinstance(subfolder_cache, dict): + for folder_uid, uids in subfolder_cache.items(): + if not isinstance(uids, set): + continue + hit = uids & protected_uid_set + if hit: + removed_from_folders[folder_uid] = hit + uids -= hit + + try: + yield + finally: + from ..decorators.logging import logger + + for attr in original_caches: + try: + restored = dict(getattr(params, attr, None) or {}) + restored.update(saved_entries[attr]) + setattr(params, attr, restored) + except Exception as e: + logger.debug(f'hide_from_record_cache: failed to restore {attr} ({type(e).__name__}); restoring protected entries only') + try: + setattr(params, attr, dict(saved_entries[attr])) + except Exception: + pass + + if isinstance(subfolder_cache, dict): + for folder_uid, hit in removed_from_folders.items(): + try: + uids = subfolder_cache.get(folder_uid) + if isinstance(uids, set): + uids |= hit + except Exception as e: + logger.debug(f'hide_from_record_cache: failed to restore subfolder {folder_uid} ({type(e).__name__})') + + +@contextlib.contextmanager +def hide_from_folder_cache(params, protected_folder_uids: Set[str]): + """For the with-block, hides protected_folder_uids from folder_cache/shared_folder_cache/subfolder_cache and + from their parent's (or root_folder's) .subfolders list, restoring everything on exit """ + if params is None or not protected_folder_uids: + yield + return + + protected_uid_set = frozenset(protected_folder_uids) + + folder_cache = getattr(params, 'folder_cache', None) + root_folder = getattr(params, 'root_folder', None) + # {uid: (subfolders_list, original_index)} -- built up incrementally inside the try below so a + # failure partway through setup still leaves whatever was already removed restorable in finally, + # rather than mutating this live list before there's any guarantee finally will run at all. + removed_from_parents: Dict[str, tuple] = {} + original_caches = {} + saved_entries = {} + + try: + if isinstance(folder_cache, dict): + for uid in protected_uid_set: + node = folder_cache.get(uid) + parent_uid = getattr(node, 'parent_uid', None) if node is not None else None + parent = folder_cache.get(parent_uid) if parent_uid else root_folder + subfolders = getattr(parent, 'subfolders', None) if parent is not None else None + if isinstance(subfolders, list) and uid in subfolders: + index = subfolders.index(uid) + subfolders.remove(uid) + removed_from_parents[uid] = (subfolders, index) + + for attr in _GUARDED_FOLDER_CACHE_ATTRS: + source = getattr(params, attr, None) + if not isinstance(source, MutableMapping): + continue + original_caches[attr] = source + saved_entries[attr] = {uid: source[uid] for uid in protected_uid_set if uid in source} + setattr(params, attr, _GuardedRecordCache(source, protected_uid_set)) + + yield + finally: + from ..decorators.logging import logger + + for attr in original_caches: + try: + restored = dict(getattr(params, attr, None) or {}) + restored.update(saved_entries[attr]) + setattr(params, attr, restored) + except Exception as e: + logger.debug(f'hide_from_folder_cache: failed to restore {attr} ({type(e).__name__}); restoring protected entries only') + try: + setattr(params, attr, dict(saved_entries[attr])) + except Exception: + pass + + for uid, (subfolders, index) in removed_from_parents.items(): + try: + if uid not in subfolders: + subfolders.insert(min(index, len(subfolders)), uid) + except Exception as e: + logger.debug(f'hide_from_folder_cache: failed to restore subfolders entry for {uid} ({type(e).__name__})') diff --git a/keepercommander/service/util/request_validation.py b/keepercommander/service/util/request_validation.py index 80d923d85..c5d9cbf32 100644 --- a/keepercommander/service/util/request_validation.py +++ b/keepercommander/service/util/request_validation.py @@ -11,6 +11,7 @@ from typing import Optional, Tuple, Dict, Any from flask import request, jsonify +from werkzeug.exceptions import BadRequest from html import escape import tempfile import os @@ -146,8 +147,14 @@ def validate_request_json() -> Optional[Tuple]: logger.info("Request validation failed: Content-Type must be application/json") return jsonify({"status": "error", "error": "Content-Type must be application/json"}), 400 - if not request.json: - logger.info("Request validation failed: Invalid or empty JSON") - return jsonify({"status": "error", "error": "Invalid or empty JSON"}), 400 - + try: + json_data = request.get_json(force=True, silent=False) + if not json_data: + logger.info("Request validation failed: Invalid or empty JSON") + return jsonify({"status": "error", "error": "Invalid or empty JSON"}), 400 + except (BadRequest, ValueError) as e: + # The parser's message can include a payload excerpt - log it, don't echo it back. + logger.warning(f"Request validation failed: JSON parsing error - {e}") + return jsonify({"status": "error", "error": "Invalid JSON format"}), 400 + return None diff --git a/keepercommander/service/util/tunneling.py b/keepercommander/service/util/tunneling.py index afc8c95d7..e3bf2dca3 100644 --- a/keepercommander/service/util/tunneling.py +++ b/keepercommander/service/util/tunneling.py @@ -20,13 +20,28 @@ import json import tempfile +from ... import utils +from .process_util import spawn_detached_process + +def get_tunnel_log_file(name): + # Resolved per call, not cached at import time, so a later --data-dir override is respected. + log_dir = os.path.join(utils.get_default_path(), "service_logs") + os.makedirs(log_dir, exist_ok=True) + return os.path.join(log_dir, name) + + +# DOA check only catches immediate crashes; a slower failure (bad auth) is caught later below. +NGROK_STARTUP_CHECK_DELAY_SECONDS = 0.5 + def start_ngrok(port, auth_token=None, subdomain=None): """ Start ngrok as a fully detached subprocess and return the PID. """ - ngrok_cmd = ["ngrok", "http", str(port), "--log=stdout", "--log-level=info"] - + ngrok_config = conf.get_default() + ngrok.install_ngrok(ngrok_config) + ngrok_cmd = [ngrok_config.ngrok_path, "http", str(port), "--log=stdout", "--log-level=info"] + if subdomain: ngrok_cmd += ["--subdomain", subdomain] if auth_token: @@ -34,37 +49,20 @@ def start_ngrok(port, auth_token=None, subdomain=None): service_core_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "core") - log_dir = os.path.join(service_core_dir, "logs") - os.makedirs(log_dir, exist_ok=True) - log_file = os.path.join(log_dir, "ngrok_subprocess.log") - - if sys.platform == "win32": - subprocess.DETACHED_PROCESS = 0x00000008 - with open(log_file, 'w') as log_f: - process = subprocess.Popen( - ngrok_cmd, - creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP, - stdout=log_f, - stderr=subprocess.STDOUT, # Combine stderr with stdout - cwd=service_core_dir, # Set working directory - env=os.environ.copy() # Inherit environment variables - ) - else: - with open(log_file, 'w') as log_f: - process = subprocess.Popen( - ngrok_cmd, - stdout=log_f, - stderr=subprocess.STDOUT, # Combine stderr with stdout - preexec_fn=os.setpgrp, - cwd=service_core_dir, # Set working directory - env=os.environ.copy() # Inherit environment variables - ) + log_file = get_tunnel_log_file("ngrok_subprocess.log") + process = spawn_detached_process(ngrok_cmd, log_file, cwd=service_core_dir, env=os.environ.copy()) + utils.set_file_permissions(log_file) + + time.sleep(NGROK_STARTUP_CHECK_DELAY_SECONDS) + if process.poll() is not None: + raise RuntimeError( + f"ngrok exited immediately (exit code {process.returncode}); see {log_file} for details" + ) actual_ngrok_pid = process.pid try: import psutil - time.sleep(0.5) # Give ngrok a moment to start - + # Look for the actual ngrok binary process for proc in psutil.process_iter(['pid', 'ppid', 'name', 'cmdline']): try: @@ -166,15 +164,22 @@ def start_ngrok_with_url(port, auth_token=None, subdomain=None): # If API method fails, try parsing the log file if not public_url: - service_core_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "core") - log_file = os.path.join(service_core_dir, "logs", "ngrok_subprocess.log") - public_url = get_ngrok_url_from_log(log_file) + public_url = get_ngrok_url_from_log(get_tunnel_log_file("ngrok_subprocess.log")) # If we still don't have a URL and subdomain was provided, construct it if not public_url and subdomain: public_url = f"https://{subdomain}.ngrok.io" logging.warning("Could not retrieve dynamic ngrok URL, using constructed URL") - + + # No URL and the process is gone (e.g. bad auth token) is a real failure, not just slow. + if not public_url: + try: + import psutil + if not psutil.pid_exists(pid): + raise RuntimeError(f"ngrok process {pid} is no longer running; see logs for details") + except ImportError: + pass + return pid, public_url def generate_ngrok_url(port, auth_token, ngrok_custom_domain, run_mode): @@ -200,32 +205,49 @@ def generate_ngrok_url(port, auth_token, ngrok_custom_domain, run_mode): log_event_callback=None, ) - with open(os.devnull, 'w') as devnull: + if run_mode == "background": + # Own subprocess with its own log file - skip the console-unsafe fd redirection below. + if ngrok_custom_domain: + ngrok_pid, public_url = start_ngrok_with_url(port=port, auth_token=auth_token, subdomain=ngrok_custom_domain) + else: + ngrok_pid, public_url = start_ngrok_with_url(port=port, auth_token=auth_token) + return public_url, ngrok_pid + + old_stdout_fd = None + old_stderr_fd = None + try: old_stdout_fd = os.dup(1) old_stderr_fd = os.dup(2) - os.dup2(devnull.fileno(), 1) - os.dup2(devnull.fileno(), 2) - + devnull_fd = os.open(os.devnull, os.O_WRONLY) try: - if run_mode == "background": - # Background mode: use subprocess for both custom and non-custom domains - if ngrok_custom_domain: - ngrok_pid, public_url = start_ngrok_with_url(port=port, auth_token=auth_token, subdomain=ngrok_custom_domain) - else: - ngrok_pid, public_url = start_ngrok_with_url(port=port, auth_token=auth_token) - return public_url, ngrok_pid - else: - # Foreground mode: use pyngrok library - if ngrok_custom_domain: - tunnel = ngrok.connect(port, subdomain=ngrok_custom_domain, pyngrok_config=ngrok_config) - else: - tunnel = ngrok.connect(port, pyngrok_config=ngrok_config) - return tunnel.public_url, None - + os.dup2(devnull_fd, 1) + os.dup2(devnull_fd, 2) finally: + os.close(devnull_fd) + except OSError: + # Restore anything already redirected (else fd 1 could stay pointed at devnull forever). + for fd, target in ((old_stdout_fd, 1), (old_stderr_fd, 2)): + if fd is not None: + try: + os.dup2(fd, target) + finally: + os.close(fd) + old_stdout_fd = None + old_stderr_fd = None + + try: + if ngrok_custom_domain: + tunnel = ngrok.connect(port, subdomain=ngrok_custom_domain, pyngrok_config=ngrok_config) + else: + tunnel = ngrok.connect(port, pyngrok_config=ngrok_config) + return tunnel.public_url, None + + finally: + if old_stdout_fd is not None: os.dup2(old_stdout_fd, 1) - os.dup2(old_stderr_fd, 2) os.close(old_stdout_fd) + if old_stderr_fd is not None: + os.dup2(old_stderr_fd, 2) os.close(old_stderr_fd) @@ -236,14 +258,15 @@ def _download_cloudflared(): Download cloudflared binary if not available. Returns path to cloudflared binary. """ - try: - # First try to find existing cloudflared - result = subprocess.run(['which', 'cloudflared'], capture_output=True, text=True) - if result.returncode == 0: - return result.stdout.strip() - except: - pass - + # Windows `where`/shutil.which both search cwd before PATH (planted-binary risk) - only search on POSIX. + if sys.platform != "win32": + try: + result = subprocess.run(['which', 'cloudflared'], capture_output=True, text=True) + if result.returncode == 0: + return result.stdout.strip().splitlines()[0] + except Exception as e: + logging.debug(f"Could not find existing cloudflared on PATH: {e}") + # Download cloudflared binary import platform import urllib.request @@ -326,32 +349,10 @@ def _start_cloudflare_with_binary(port, tunnel_token, custom_domain=None): ) service_core_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "core") - log_dir = os.path.join(service_core_dir, "logs") - os.makedirs(log_dir, exist_ok=True) - log_file = os.path.join(log_dir, "cloudflare_tunnel_subprocess.log") - - if sys.platform == "win32": - subprocess.DETACHED_PROCESS = 0x00000008 - with open(log_file, 'w') as log_f: - process = subprocess.Popen( - cloudflared_cmd, - creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP, - stdout=log_f, - stderr=subprocess.STDOUT, - cwd=service_core_dir, - env=os.environ.copy() - ) - else: - with open(log_file, 'w') as log_f: - process = subprocess.Popen( - cloudflared_cmd, - stdout=log_f, - stderr=subprocess.STDOUT, - preexec_fn=os.setpgrp, - cwd=service_core_dir, - env=os.environ.copy() - ) - + log_file = get_tunnel_log_file("cloudflare_tunnel_subprocess.log") + process = spawn_detached_process(cloudflared_cmd, log_file, cwd=service_core_dir, env=os.environ.copy()) + utils.set_file_permissions(log_file) + tunnel_url = get_cloudflare_url_from_log(log_file, custom_domain) return process.pid, tunnel_url @@ -380,9 +381,9 @@ def get_cloudflare_url_from_log(log_file, custom_domain=None, max_retries=10, re matches = re.findall(pattern, content) for match in matches: # Filter out localhost and other non-public URLs - if ('localhost' not in match and - '127.0.0.1' not in match and - 'trycloudflare.com' in match or 'cfargotunnel.com' in match or custom_domain in match if custom_domain else True): + if ('localhost' not in match and '127.0.0.1' not in match and + ('trycloudflare.com' in match or 'cfargotunnel.com' in match + or (custom_domain and custom_domain in match))): return match except Exception as e: @@ -410,36 +411,22 @@ def start_cloudflare_tunnel_with_url(port, tunnel_token, custom_domain=None): def generate_cloudflare_url(port, tunnel_token, custom_domain, run_mode): """ - Start a Cloudflare tunnel with complete log suppression. - Returns a tuple of (public_url, tunnel_pid) for background mode, or (public_url, None) for foreground mode. + Start a Cloudflare tunnel as a detached subprocess and return its public URL. + Returns a tuple of (public_url, tunnel_pid). """ if not port: raise ValueError("Port must be provided for Cloudflare tunnel.") - + if not tunnel_token or not tunnel_token.strip(): raise ValueError( "Tunnel token is required for secure Cloudflare tunnel operation. " "Temporary tunnels are not supported for production use." ) - - # Cloudflare tunnel configuration - - with open(os.devnull, 'w') as devnull: - old_stdout_fd = os.dup(1) - old_stderr_fd = os.dup(2) - os.dup2(devnull.fileno(), 1) - os.dup2(devnull.fileno(), 2) - - try: - tunnel_pid, public_url = start_cloudflare_tunnel_with_url( - port=port, - tunnel_token=tunnel_token, - custom_domain=custom_domain - ) - return public_url, tunnel_pid - - finally: - os.dup2(old_stdout_fd, 1) - os.dup2(old_stderr_fd, 2) - os.close(old_stdout_fd) - os.close(old_stderr_fd) \ No newline at end of file + + # Always runs as a detached subprocess with its own log file - nothing to suppress here. + tunnel_pid, public_url = start_cloudflare_tunnel_with_url( + port=port, + tunnel_token=tunnel_token, + custom_domain=custom_domain + ) + return public_url, tunnel_pid diff --git a/keepercommander/service/util/verified_command.py b/keepercommander/service/util/verified_command.py index 20bbb6fcb..bd5238d29 100644 --- a/keepercommander/service/util/verified_command.py +++ b/keepercommander/service/util/verified_command.py @@ -17,6 +17,10 @@ class Verifycommand: # Aliases from record.py — CommandExecutor checks tokens before cli expands them. _RECORD_EDIT_COMMANDS = frozenset({'record-add', 'ra', 'record-update', 'ru'}) + _PROTECTED_RECORD_MSG = ( + 'Service Mode configuration records are not accessible through Service Mode' + ) + # Legacy Commands category — plugin-based rotation/connection commands have no safe Service Mode form # and are blocked unconditionally, regardless of what an API key's command_list allows. _LEGACY_COMMANDS = frozenset({ @@ -36,6 +40,11 @@ class Verifycommand: "The '--' argument separator is not permitted through Service Mode" ) + # expand_cmd_args (commands/base.py) substitutes ${VARNAME} after this check runs but before execution -- ban it outright. + _ENV_VAR_EXPANSION_MSG = ( + "The '${VARNAME}' environment-variable syntax is not permitted through Service Mode" + ) + # WARNING: everything below is a DENYLIST. Any command/flag that reads or # writes a host file and is NOT enumerated here is allowed by default. # Adding a new command with local file I/O? Add it here, or it silently @@ -85,6 +94,7 @@ def validate_service_mode_restrictions(command_tokens, request_temp_dir=None): for validator in ( Verifycommand.validate_service_mode_double_dash, + Verifycommand.validate_service_mode_env_var_expansion_command, Verifycommand.validate_service_mode_legacy_command, Verifycommand.validate_service_mode_pam_tunnel_command, Verifycommand.validate_service_mode_download_attachment_command, @@ -99,6 +109,33 @@ def validate_service_mode_restrictions(command_tokens, request_temp_dir=None): return error return None + @staticmethod + def _record_reference_candidates(tok): + """tok itself, plus its value if tok is a --flag=value (or -f=value) option.""" + if '=' in tok: + _, _, value = tok.partition('=') + if value: + return (tok, value) + return (tok,) + + @staticmethod + def validate_service_mode_protected_record_command(command_tokens, protected_uids=None): + """Reject any command with a protected title/UID as a whole token or --flag=value; indirect forms (comma lists, path-qualified titles) rely on protected_records.hide_from_record_cache instead.""" + if not command_tokens: + return None + + from .protected_records import get_protected_record_title_set + protected_titles = get_protected_record_title_set() + + uid_set = set(protected_uids) if protected_uids else set() + for tok in command_tokens[1:]: + for candidate in Verifycommand._record_reference_candidates(tok): + if candidate.lower() in protected_titles: + return Verifycommand._PROTECTED_RECORD_MSG + if candidate in uid_set: + return Verifycommand._PROTECTED_RECORD_MSG + return None + @staticmethod def validate_service_mode_double_dash(command_tokens, request_temp_dir=None): """Block bare '--' anywhere in Service Mode input; error or None.""" @@ -106,6 +143,14 @@ def validate_service_mode_double_dash(command_tokens, request_temp_dir=None): return Verifycommand._DOUBLE_DASH_MSG return None + @staticmethod + def validate_service_mode_env_var_expansion_command(command_tokens, request_temp_dir=None): + """Block '${VARNAME}' anywhere in Service Mode input; error or None.""" + from ...commands.base import parameter_pattern + if any(parameter_pattern.search(tok) for tok in command_tokens): + return Verifycommand._ENV_VAR_EXPANSION_MSG + return None + @staticmethod def validate_service_mode_legacy_command(command_tokens, request_temp_dir=None): """Block legacy commands in Service Mode; error or None.""" diff --git a/tests/test_cyberark_pam_import.py b/tests/test_cyberark_pam_import.py index 019677715..1b4352b55 100644 --- a/tests/test_cyberark_pam_import.py +++ b/tests/test_cyberark_pam_import.py @@ -1,18 +1,27 @@ -# _ __ -# | |/ /___ ___ _ __ ___ _ _ ® +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® # | ' monthly def test_matching_interval_without_flag_inherits_master(self): - """No flag, and the interval matches master — no exception exists.""" + """No flag, and the interval matches master — no exception exists.""" client = MagicMock() client.fetch_platform_rotation_policy.return_value = { "change": {"interval": 90, "allowedPeriodic": True}, @@ -3571,7 +4065,7 @@ def test_returns_none_on_network_error(self, mock_dns, mock_requests): assert result is None -# ── Red Team Coverage Tests ────────────────────────────────── +# ── Red Team Coverage Tests ────────────────────────────────── class TestEscFunction: """Tests for _esc() HTML + control char sanitizer.""" @@ -3691,7 +4185,7 @@ def test_fetch_users_respects_cap(self, mock_dns, mock_requests): client = CyberArkPVWAClient("pvwa.example.com") client.auth_token = "test" result = client.fetch_users() - # Should stop well before 999999 — capped at MAX_FETCH_RECORDS + # Should stop well before 999999 — capped at MAX_FETCH_RECORDS assert len(result) <= MAX_FETCH_RECORDS + 100 # allow 1 page overshoot @@ -3719,7 +4213,7 @@ def mock_get(url, **kwargs): "nextLink": "https://attacker.example.com/steal?token=x", } else: - # If we DID follow it, we'd hit this — test asserts we don't + # If we DID follow it, we'd hit this — test asserts we don't resp.json.return_value = {"Users": [], "nextLink": None} return resp @@ -3775,7 +4269,7 @@ def test_returns_429_after_max_retries(self, mock_dns, mock_requests, mock_sleep assert mock_requests.get.call_count == 3 # MAX_RETRIES -# ── Phase 7 Tests: Enhanced Report + Cleanup ───────────────── +# ── Phase 7 Tests: Enhanced Report + Cleanup ───────────────── class TestEnhancedReport: """Tests for the enhanced build_report with all sections.""" @@ -3803,13 +4297,13 @@ def test_report_has_all_sections(self, caplog): assert "IMPORT RESULTS" in report assert "PLATFORM MAPPING" in report assert "SKIPPED ACCOUNTS" in report - assert "UNMAPPED — REQUIRES MANUAL ACTION" not in report + assert "UNMAPPED — REQUIRES MANUAL ACTION" not in report assert "Dual control" not in report assert "debug log" in report assert "NEXT STEPS" in report assert "COMMAND" in report assert "pvwa.example.com" in report - assert "UNMAPPED — REQUIRES MANUAL ACTION" in caplog.text + assert "UNMAPPED — REQUIRES MANUAL ACTION" in caplog.text assert "Dual control" in caplog.text def test_report_gateway_token(self): @@ -3852,14 +4346,12 @@ class TestCleanupCommand: """Tests for CyberArkPAMCleanupCommand.""" def test_missing_args_raises(self): - from keepercommander.commands.pam_import.cyberark_import import CyberArkPAMCleanupCommand from keepercommander.error import CommandError cmd = CyberArkPAMCleanupCommand() with pytest.raises(CommandError): cmd.execute(MagicMock(), project_name="", config_uid="") def test_parser_has_flags(self): - from keepercommander.commands.pam_import.cyberark_import import CyberArkPAMCleanupCommand cmd = CyberArkPAMCleanupCommand() args = cmd.parser.parse_args(["--name", "Test", "--dry-run", "--yes"]) assert args.project_name == "Test" @@ -3867,12 +4359,124 @@ def test_parser_has_flags(self): assert args.auto_confirm is True def test_parser_config_flag(self): - from keepercommander.commands.pam_import.cyberark_import import CyberArkPAMCleanupCommand cmd = CyberArkPAMCleanupCommand() args = cmd.parser.parse_args(["--config", "uid123"]) assert args.config_uid == "uid123" +class TestCyberArkImportNsfSupport: + """NSF (--nsf) wiring for cyberark-import / cleanup / discovery.""" + + def test_single_batch_passes_use_nsf_to_import(self): + cmd = CyberArkPAMImportCommand() + params = MagicMock() + import_data = {"pam_data": {"resources": [], "users": []}} + + with patch.object(_temp_store, "write_json", return_value="/tmp/x.json"), \ + patch.object(_temp_store, "remove"), \ + patch("keepercommander.commands.pam_import.edit.PAMProjectImportCommand") as mock_import, \ + patch.object(cmd, "_find_config_uid", return_value="cfg-uid"): + mock_import.return_value.execute.return_value = None + result = cmd._single_batch_import( + params, import_data, "Proj", "", use_nsf=True, + ) + + assert result["config_uid"] == "cfg-uid" + kwargs = mock_import.return_value.execute.call_args.kwargs + assert kwargs.get("use_nsf") is True + assert kwargs.get("project_name") == "Proj" + + def test_single_batch_extend_ignores_use_nsf_flag(self): + cmd = CyberArkPAMImportCommand() + params = MagicMock() + import_data = {"pam_data": {"resources": [], "users": []}} + + with patch.object(_temp_store, "write_json", return_value="/tmp/x.json"), \ + patch.object(_temp_store, "remove"), \ + patch("keepercommander.commands.pam_import.extend.PAMProjectExtendCommand") as mock_extend: + mock_extend.return_value.execute.return_value = None + result = cmd._single_batch_import( + params, import_data, "Proj", "existing-cfg", use_nsf=True, + ) + + assert result["config_uid"] == "existing-cfg" + kwargs = mock_extend.return_value.execute.call_args.kwargs + assert "use_nsf" not in kwargs + assert kwargs.get("config") == "existing-cfg" + + def test_find_nsf_project_wrapper_uids(self): + root = NestedShareFolderNode() + root.uid = "nsf-root" + root.name = PAM_ROOT_FOLDER_NAME + root.parent_uid = None + root.subfolders = ["nsf-proj"] + + proj = NestedShareFolderNode() + proj.uid = "nsf-proj" + proj.name = DEFAULT_PROJECT_NAME + proj.parent_uid = "nsf-root" + proj.subfolders = ["nsf-safe"] + + safe = NestedShareFolderNode() + safe.uid = "nsf-safe" + safe.name = SAFE_FOLDER_NAME + safe.parent_uid = "nsf-proj" + safe.subfolders = [] + + params = SimpleNamespace( + folder_cache={ + "nsf-root": root, + "nsf-proj": proj, + "nsf-safe": safe, + }, + nested_share_folders={ + "nsf-root": {"name": PAM_ROOT_FOLDER_NAME, "parent_uid": None}, + "nsf-proj": {"name": DEFAULT_PROJECT_NAME, "parent_uid": "nsf-root"}, + "nsf-safe": {"name": SAFE_FOLDER_NAME, "parent_uid": "nsf-proj"}, + }, + shared_folder_cache={}, + subfolder_record_cache={}, + nested_share_folder_records={"nsf-safe": {"rec-1"}}, + ) + + wrappers = CyberArkPAMCleanupCommand._find_project_wrapper_folder_uids( + params, DEFAULT_PROJECT_NAME, + ) + assert wrappers == ["nsf-proj"] + + children = list(CyberArkPAMCleanupCommand._iter_project_child_folders( + params, "nsf-proj", + )) + assert children == [("nsf-safe", SAFE_FOLDER_NAME)] + + def test_find_classic_wrapper_still_works(self): + root = SimpleNamespace( + uid="uf-root", name=PAM_ROOT_FOLDER_NAME, parent_uid=None, + type=BaseFolderNode.UserFolderType, subfolders=["uf-proj"], + ) + proj = SimpleNamespace( + uid="uf-proj", name="MyProj", parent_uid="uf-root", + type=BaseFolderNode.UserFolderType, subfolders=["sf-safe"], + ) + safe = SimpleNamespace( + uid="sf-safe", name="SafeA", parent_uid="uf-proj", + type=BaseFolderNode.SharedFolderType, subfolders=[], + ) + params = SimpleNamespace( + folder_cache={"uf-root": root, "uf-proj": proj, "sf-safe": safe}, + nested_share_folders={}, + shared_folder_cache={}, + ) + wrappers = CyberArkPAMCleanupCommand._find_project_wrapper_folder_uids( + params, "MyProj", + ) + assert wrappers == ["uf-proj"] + children = list(CyberArkPAMCleanupCommand._iter_project_child_folders( + params, "uf-proj", + )) + assert children == [("sf-safe", "SafeA")] + + class TestSSHKeyImport: """C4: SSH key platforms store private key in private_pem_key, not password.""" @@ -4016,7 +4620,7 @@ def test_unmatched_members_excluded_from_permissions(self): } matcher = UserTeamMatcher(keeper_users=[{"email": "admin@corp.com"}]) result = build_shared_folder_permissions(safe_members, matcher) - assert result == {} # No matched members → no shared folder perms + assert result == {} # No matched members → no shared folder perms def test_team_members_matched(self): from keepercommander.importer.cyberark.cyberark_pam import build_shared_folder_permissions @@ -4075,13 +4679,13 @@ def test_empty_platform_id_uses_fallback(self): } result = mapper.map_account(account, None) assert result is not None - # No address + no platform → routed to login (not an unreachable pamMachine) + # No address + no platform → routed to login (not an unreachable pamMachine) assert result["type"] == "login" assert mapper.unmapped_platforms.get("(empty)", 0) == 1 def test_no_address_account_routed_to_login(self): """CyberArk accounts without an address can't be PAM resources - (gateway has nothing to connect to) — import as login.""" + (gateway has nothing to connect to) — import as login.""" mapper = AccountMapper() account = { "id": "1", "name": "floating-cred", "platformId": "UnixSSH", @@ -4092,7 +4696,7 @@ def test_no_address_account_routed_to_login(self): assert result["type"] == "login" assert result["login"] == "svc" assert result["password"] == "s3cret" - assert "No address" in result["notes"] + assert "notes" not in result def test_no_address_ssh_key_stays_pam_machine(self): """SSH keys without address stay as pamMachine so the @@ -4108,7 +4712,7 @@ def test_no_address_ssh_key_stays_pam_machine(self): assert result["type"] == "pamMachine" assert result["users"][0]["private_pem_key"] == fake_key - def test_missing_platform_id_uses_fallback(self): + def test_missing_platform_id_without_pattern_maps_to_login(self): mapper = AccountMapper() account = { "id": "36_3", "name": "PSMServer", @@ -4116,7 +4720,8 @@ def test_missing_platform_id_uses_fallback(self): } result = mapper.map_account(account, "pass") assert result is not None - assert result["type"] == "pamMachine" + assert result["type"] == "login" + assert "users" not in result def test_generic_network_account_maps_to_ssh(self): mapper = AccountMapper() @@ -4161,7 +4766,7 @@ def test_logon_domain_not_set_for_databases(self): result = mapper.map_account(account, "pass") assert "domain_name" not in result # Only pamMachine gets domain_name - def test_cpm_failure_status_annotated(self): + def test_cpm_failure_status_does_not_write_notes(self): mapper = AccountMapper() account = { "id": "25_7", "name": "x_accountB", "platformId": "WinDesktopLocal", @@ -4174,8 +4779,7 @@ def test_cpm_failure_status_annotated(self): } result = mapper.map_account(account, "pass") user = result["users"][0] - assert "FAILURE" in user.get("notes", "") - assert "(CPM)MaxRetries" in user.get("notes", "") + assert "notes" not in user def test_system_safes_expanded(self): from keepercommander.importer.cyberark.cyberark_pam import SYSTEM_SAFES @@ -4204,7 +4808,7 @@ class TestFolderDeduplication: def test_ksm_mode_dedup_adds_suffix(self): mapper = SafeFolderMapper(mode="ksm") name1 = mapper.map_safe("IT Safe", "proj") - name2 = mapper.map_safe("IT Safe!", "proj") # "!" stripped → "IT Safe" collision + name2 = mapper.map_safe("IT Safe!", "proj") # "!" stripped → "IT Safe" collision assert name1 != name2 assert "#2" in name2 @@ -4286,17 +4890,17 @@ def test_default_without_policy(self): assert cfg["text_session_recording"] == "off" -# ═══════════════════════════════════════════════════════════════════════ -# INTEGRATION TESTS — Real CyberArk PVWA JSON → Keeper Vault JSON -# ═══════════════════════════════════════════════════════════════════════ +# ═══════════════════════════════════════════════════════════════════════ +# INTEGRATION TESTS — Real CyberArk PVWA JSON → Keeper Vault JSON +# ═══════════════════════════════════════════════════════════════════════ # These tests use actual CyberArk API response shapes captured from a -# real on-prem PVWA environment. They verify the full input→output -# pipeline: API response → AccountMapper → SafeFolderMapper → -# validate_import_data → build_import_json → vault-ready JSON. +# real on-prem PVWA environment. They verify the full input→output +# pipeline: API response → AccountMapper → SafeFolderMapper → +# validate_import_data → build_import_json → vault-ready JSON. # Real CyberArk PVWA /api/Accounts response (subset from on-prem env) REAL_PVWA_ACCOUNTS = [ - # Unix SSH — CPM enabled, standard account + # Unix SSH — CPM enabled, standard account {"id": "28_11", "name": "Operating System-UnixSSH-10.0.1.30-simon", "platformId": "UnixSSH", "safeName": "partner", "address": "10.0.1.30", "userName": "simon", "secretType": "password", @@ -4305,7 +4909,7 @@ def test_default_without_policy(self): "lastModifiedTime": 1674712329}, "createdTime": 1674712329}, - # MSSQL Database — CPM disabled, custom port + database + # MSSQL Database — CPM disabled, custom port + database {"id": "25_15", "name": "db1", "platformId": "MSSql", "safeName": "Test", "address": "dbserver1.cyberark.local", "userName": "sa", "secretType": "password", @@ -4314,7 +4918,7 @@ def test_default_without_policy(self): "manualManagementReason": "NoReason"}, "createdTime": 1563922540}, - # Windows Desktop — CPM enabled, LogonDomain set + # Windows Desktop — CPM enabled, LogonDomain set {"id": "25_3", "name": "windows1", "platformId": "WinDesktopLocal", "safeName": "Test", "address": "components", "userName": "svc_account", "secretType": "password", @@ -4323,7 +4927,7 @@ def test_default_without_policy(self): "status": "success"}, "createdTime": 1551300856}, - # SSH Keys — CPM disabled, secretType=key + # SSH Keys — CPM disabled, secretType=key {"id": "25_14", "name": "Linux2", "platformId": "UnixSSHKeys", "safeName": "Test", "address": "linux2.cyberark.local", "userName": "root", "secretType": "key", "platformAccountProperties": {}, @@ -4331,7 +4935,7 @@ def test_default_without_policy(self): "manualManagementReason": "NoReason"}, "createdTime": 1563922244}, - # Windows — CPM failure status + # Windows — CPM failure status {"id": "25_7", "name": "Operating System-WindowsDesktopLocalAccountsRotationalPolicy-10.0.1.20-x_accountB", "platformId": "WinDesktopLocal", "safeName": "Test", @@ -4342,7 +4946,7 @@ def test_default_without_policy(self): "status": "failure"}, "createdTime": 1551306296}, - # BusinessWebsite → login record (not pamMachine) + # BusinessWebsite → login record (not pamMachine) {"id": "25_5", "name": "web-portal", "platformId": "BusinessWebsite", "safeName": "Test", "address": "", "userName": "admin", "platformAccountProperties": {"URL": "https://portal.company.com"}, @@ -4385,7 +4989,7 @@ def test_default_without_policy(self): "secretManagement": {"automaticManagementEnabled": True}, "createdTime": 1563908159}, - # Oracle Database — custom port + database + # Oracle Database — custom port + database {"id": "25_16", "name": "db2", "platformId": "Oracle", "safeName": "Test", "address": "dbserver2.cyberark.local", "userName": "oradb", "secretType": "password", @@ -4394,7 +4998,7 @@ def test_default_without_policy(self): "manualManagementReason": "NoReason"}, "createdTime": 1563922714}, - # Empty platformId — should use fallback + # Empty platformId — should use fallback {"id": "25_25", "name": "testobject", "platformId": "", "safeName": "Test", "address": "", "userName": "", "secretType": "password", "platformAccountProperties": {}, @@ -4415,7 +5019,7 @@ def _run_full_pipeline(accounts=None, project_name="CyberArk-Test", filtered = exclude_system_safes(all_safes) ok_safes = {s["safeName"] for s in filtered} - # Step 2: Map accounts → resources + users + # Step 2: Map accounts → resources + users mapper = AccountMapper() folder_mapper = SafeFolderMapper(mode=folder_mode) resources, users, skipped = [], [], [] @@ -4457,7 +5061,7 @@ def _run_full_pipeline(accounts=None, project_name="CyberArk-Test", class TestEndToEndPipeline: - """Integration: real PVWA JSON → full vault import JSON.""" + """Integration: real PVWA JSON → full vault import JSON.""" def test_overall_structure(self): data, _, _, _ = _run_full_pipeline() @@ -4490,7 +5094,7 @@ def test_resource_count(self): data, _, _, _ = _run_full_pipeline() resources = data["pam_data"]["resources"] # 12 accounts - 2 system safe - 1 login = 9 resources - # BUT empty platformId testobject has no address → still created as resource + # BUT empty platformId testobject has no address → still created as resource assert len(resources) >= 8 def test_login_records_separated(self): @@ -4548,7 +5152,7 @@ def test_mssql_database_resource(self): u = r["users"][0] assert u["connect_database"] == "hr" assert u["rotation_settings"]["enabled"] == "off" - assert "CPM disabled" in u.get("notes", "") + assert "notes" not in u def test_oracle_database_resource(self): data, _, _, _ = _run_full_pipeline() @@ -4588,14 +5192,11 @@ def test_ssh_key_resource(self): assert u["password"] == "" assert u["rotation_settings"]["enabled"] == "off" - def test_cpm_failure_annotated(self): + def test_cpm_failure_does_not_write_notes(self): data, _, _, _ = _run_full_pipeline() r = self._find_resource(data, "x_accountB") u = r["users"][0] - notes = u.get("notes", "") - assert "CPM disabled" in notes - assert "(CPM)MaxRetries" in notes - assert "FAILURE" in notes + assert "notes" not in u assert u["rotation_settings"]["enabled"] == "off" def test_generic_network_device(self): @@ -4625,7 +5226,7 @@ def test_every_nested_user_is_pam_user(self): assert u["type"] == "pamUser" def test_launch_credentials_matches_user_title(self): - """edit.py resolves launch_credentials by title → UID.""" + """edit.py resolves launch_credentials by title → UID.""" data, _, _, _ = _run_full_pipeline() for r in data["pam_data"]["resources"]: ps = r.get("pam_settings", {}) @@ -4644,7 +5245,7 @@ def test_rotation_settings_resource_field(self): rs = u.get("rotation_settings", {}) if rs.get("rotation") == "general": # edit.py auto-sets resourceUid to parent machine UID - # We don't set it here — edit.py handles it at import time + # We don't set it here — edit.py handles it at import time assert rs["rotation"] == "general" def test_managed_flag_on_users_with_password(self): @@ -4656,7 +5257,7 @@ def test_managed_flag_on_users_with_password(self): class TestFolderAssignment: - """Verify folder_path flows from safe → resource + nested users.""" + """Verify folder_path flows from safe → resource + nested users.""" def test_folder_path_on_resources(self): data, _, _, _ = _run_full_pipeline(folder_mode="ksm") @@ -4683,7 +5284,7 @@ def test_nested_user_folder_under_users_root(self): assert fp.startswith("CyberArk-Test - Users/"), f"Nested user folder not under Users root: {fp}" def test_resource_and_user_share_safe_subfolder(self): - """Resource in 'Test' safe → Resources/Test, its user → Users/Test.""" + """Resource in 'Test' safe → Resources/Test, its user → Users/Test.""" data, _, _, _ = _run_full_pipeline(folder_mode="ksm") for r in data["pam_data"]["resources"]: res_safe = r["folder_path"].split("/", 1)[1] if "/" in r["folder_path"] else "" @@ -4714,12 +5315,12 @@ def test_real_data_warnings(self): def test_no_address_account_becomes_login(self): """testobject (no address) is routed to a login record rather than - an unreachable pamMachine — validated at the data level.""" + an unreachable pamMachine — validated at the data level.""" data, _, _, _ = _run_full_pipeline() logins = data["pam_data"]["users"] testobject = next((l for l in logins if l["title"] == "testobject"), None) assert testobject is not None - assert "no address" in testobject.get("notes", "").lower() + assert "notes" not in testobject def test_clean_data_no_warnings(self): clean = [REAL_PVWA_ACCOUNTS[0]] # Just the Unix SSH account @@ -4811,7 +5412,7 @@ def test_output_is_json_serializable(self): assert len(roundtrip["pam_data"]["resources"]) == len(data["pam_data"]["resources"]) -# ── Optional local PVWA sample under .sample-data/ (gitignored; skip if absent) ── +# ── Optional local PVWA sample under .sample-data/ (gitignored; skip if absent) ── _SAMPLE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".sample-data")) _SAMPLE_FILES = ("safes.json", "accounts.json", "passwords.json") @@ -4911,7 +5512,7 @@ def test_vault_json_builds_and_serializes(self): assert "pam_data" in data -# ── Record-kind discriminator + ApplicationMapper stub ─────── +# ── Record-kind discriminator + ApplicationMapper stub ─────── class TestRecordKindDiscriminator: @@ -4993,9 +5594,9 @@ def test_custom_record_type_keeps_unmapped_cyberark_data_as_custom_fields(self): CyberArkImporter._add_account_metadata(record, { "platformId": "GenericPlatform", "platformAccountProperties": { - "OwnerName": "service-owner", + "OwnerName": "sample-owner", "Protocol": "SSH", - "Environment": {"Name": "prod"}, + "Environment": {"Name": "sample-env"}, }, }) @@ -5015,9 +5616,9 @@ def test_custom_record_type_keeps_unmapped_cyberark_data_as_custom_fields(self): assert custom_by_type["url"] == "https://server.example.com" assert custom_by_type["host"] == {"hostName": "server.example.com", "port": "22"} assert custom_by_label["Platform Name"] == "GenericPlatform" - assert custom_by_label["Owner Name"] == "service-owner" + assert custom_by_label["Owner Name"] == "sample-owner" assert custom_by_label["Protocol"] == "SSH" - assert custom_by_label["Environment.Name"] == "prod" + assert custom_by_label["Environment.Name"] == "sample-env" def test_import_engine_forwards_record_type_to_importer(self, monkeypatch): from keepercommander.importer import imp_exp @@ -5044,3 +5645,4 @@ def execute(self, filename, **kwargs): assert captured["filename"] == "https://pvwa.example.com" assert captured["kwargs"]["record_type"] == "test custom" + diff --git a/unit-tests/data_enterprise.py b/unit-tests/data_enterprise.py index e59cc10c8..230f9430e 100644 --- a/unit-tests/data_enterprise.py +++ b/unit-tests/data_enterprise.py @@ -26,6 +26,10 @@ _ROLE1_ID = (_ENTERPRISE_ID << 32) + 301 _ROLE1_NAME = 'Role 1' +_ROLE2_ID = (_ENTERPRISE_ID << 32) + 302 +_ROLE2_NAME = 'Role 2' +_ROLE_ADMIN_ID = (_ENTERPRISE_ID << 32) + 303 +_ROLE_ADMIN_NAME = 'Admin Role' _LAST_ID = 1000 @@ -45,6 +49,10 @@ def __init__(self): self.user2_email = _USER2_EMAIL self.role1_id = _ROLE1_ID self.role1_name = _ROLE1_NAME + self.role2_id = _ROLE2_ID + self.role2_name = _ROLE2_NAME + self.role_admin_id = _ROLE_ADMIN_ID + self.role_admin_name = _ROLE_ADMIN_NAME def enterprise_allocate_ids(params, request): @@ -122,6 +130,22 @@ def get_enterprise_data(params): crypto.encrypt_aes_v1(json.dumps({'displayname': _ROLE1_NAME}).encode('utf-8'), _TREE_KEY)), 'visible_below': True, 'new_user_inherit': True + }, + { + 'role_id': _ROLE2_ID, + 'node_id': _NODE1_ID, + 'encrypted_data': utils.base64_url_encode( + crypto.encrypt_aes_v1(json.dumps({'displayname': _ROLE2_NAME}).encode('utf-8'), _TREE_KEY)), + 'visible_below': False, + 'new_user_inherit': False + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'node_id': _NODE1_ID, + 'encrypted_data': utils.base64_url_encode( + crypto.encrypt_aes_v1(json.dumps({'displayname': _ROLE_ADMIN_NAME}).encode('utf-8'), _TREE_KEY)), + 'visible_below': False, + 'new_user_inherit': False } ] rs['managed_nodes'] = [ @@ -129,6 +153,16 @@ def get_enterprise_data(params): 'role_id': _ROLE1_ID, 'managed_node_id': _NODE1_ID, 'cascade_node_management': True, + }, + { + 'role_id': _ROLE2_ID, + 'managed_node_id': _NODE1_ID, + 'cascade_node_management': False, + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'cascade_node_management': True, } ] rs['role_users'] = [ @@ -137,6 +171,49 @@ def get_enterprise_data(params): 'enterprise_user_id': _USER1_ID } ] + # KC-1412: Add role_privileges to test authorization checks + rs['role_privileges'] = [ + { + 'role_id': _ROLE1_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_nodes' + }, + { + 'role_id': _ROLE1_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_user' + }, + { + 'role_id': _ROLE1_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_roles' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_nodes' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_user' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_roles' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'transfer_account' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_teams' + } + ] rs['teams'] = [ { 'team_uid': _TEAM1_UID, diff --git a/unit-tests/pam/test_gateway_enforcement_gate.py b/unit-tests/pam/test_gateway_enforcement_gate.py new file mode 100644 index 000000000..ea090dbdb --- /dev/null +++ b/unit-tests/pam/test_gateway_enforcement_gate.py @@ -0,0 +1,323 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' MagicMock + """mode: 'allowed' (key present, True), 'denied_explicit_false' (key + present, False), 'denied_absent' (booleans non-empty but key missing — + what Commander's account-summary parser actually produces when a + checkbox is unchecked, since it drops `:false` entries), or + 'no_enterprise' (no enforcement context at all — personal account).""" + params = MagicMock() + if mode == 'no_enterprise': + params.enforcements = None + return params + if mode == 'allowed': + booleans = [{'key': 'allow_pam_gateway', 'value': True}] + elif mode == 'denied_explicit_false': + booleans = [{'key': 'allow_pam_gateway', 'value': False}] + elif mode == 'denied_absent': + booleans = [{'key': 'some_other_key', 'value': True}] + else: + raise ValueError(f'unknown mode: {mode}') + params.enforcements = {'booleans': booleans} + return params + + +# A real multi-key `booleans` payload (captured from a live account_summary +# response) — models a role with many unrelated PAM permissions granted, +# to confirm the key lookup isn't fooled by a "some true entries exist" +# shortcut and actually finds/omits allow_pam_gateway specifically. +_MIXED_ROLE_BOOLEANS_TEMPLATE = [ + {'key': 'send_breach_watch_events', 'value': True}, + {'key': 'allow_alternate_passwords', 'value': True}, + {'key': 'allow_pam_discovery', 'value': True}, + {'key': 'allow_pam_rotation', 'value': True}, + {'key': 'allow_configure_pam_cloud_connection_settings', 'value': True}, + {'key': 'restrict_mac_fingerprint', 'value': True}, + {'key': 'allow_launch_pam_on_cloud_connection', 'value': True}, + {'key': 'allow_configure_rbi', 'value': True}, + {'key': 'allow_launch_rbi', 'value': True}, + {'key': 'allow_secrets_manager', 'value': True}, + {'key': 'allow_launch_pam_tunnels', 'value': True}, + {'key': 'allow_configure_workflow_settings', 'value': True}, + {'key': 'allow_configure_uss_settings', 'value': True}, + {'key': 'allow_rotate_credentials', 'value': True}, + {'key': 'allow_view_kcm_recordings', 'value': True}, + {'key': 'allow_view_rbi_recordings', 'value': True}, + {'key': 'allow_configure_rotation_settings', 'value': True}, + {'key': 'allow_configure_pam_tunneling_settings', 'value': True}, + {'key': 'allow_can_edit_external_shares', 'value': True}, +] + + +def _params_with_mixed_role(gateway_allowed): + # type: (bool) -> MagicMock + """A role holding ~19 unrelated PAM/vault permissions, with + allow_pam_gateway either granted or omitted alongside them — the + real-world "partial permissions" shape, as opposed to a single-key + toy list.""" + params = MagicMock() + booleans = list(_MIXED_ROLE_BOOLEANS_TEMPLATE) + if gateway_allowed: + booleans = booleans + [{'key': 'allow_pam_gateway', 'value': True}] + params.enforcements = {'booleans': booleans} + return params + + +class TestPAMCreateGatewayCommandEnforcement(unittest.TestCase): + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_denied_when_enforcement_absent(self, mock_get_app_record, mock_create_gateway): + params = _params_with_enforcement('denied_absent') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + result = PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertIsNone(result) + self.assertFalse(mock_get_app_record.called) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_denied_when_enforcement_explicit_false(self, mock_get_app_record, mock_create_gateway): + params = _params_with_enforcement('denied_explicit_false') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + result = PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertIsNone(result) + self.assertFalse(mock_get_app_record.called) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_ksm_app_display_info') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_allowed_when_enforcement_true( + self, mock_get_app_record, mock_get_ksm_app_display_info, mock_create_gateway): + params = _params_with_enforcement('allowed') + mock_get_app_record.return_value = {'record_uid': 'app_uid'} + mock_get_ksm_app_display_info.return_value = ('PAM_application', True, 'PAM_application (app_uid)') + mock_create_gateway.return_value = 'US:one-time-token' + + PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application', return_value=True) + + self.assertTrue(mock_create_gateway.called) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_ksm_app_display_info') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_allowed_when_no_enforcements( + self, mock_get_app_record, mock_get_ksm_app_display_info, mock_create_gateway): + # Personal / non-enterprise account: no enforcement context, fail open. + params = _params_with_enforcement('no_enterprise') + mock_get_app_record.return_value = {'record_uid': 'app_uid'} + mock_get_ksm_app_display_info.return_value = ('PAM_application', True, 'PAM_application (app_uid)') + mock_create_gateway.return_value = 'US:one-time-token' + + PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application', return_value=True) + + self.assertTrue(mock_create_gateway.called) + + +class TestPAMGatewayRemoveCommandEnforcement(unittest.TestCase): + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + def test_denied_when_enforcement_absent(self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('denied_absent') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + def test_denied_when_enforcement_explicit_false(self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('denied_explicit_false') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation.utils.base64_url_encode') + def test_allowed_when_enforcement_true( + self, mock_base64_url_encode, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('allowed') + gateway = MagicMock(controllerUid=b'controller_uid', controllerName='pocbyuser') + mock_get_all_gateways.return_value = [gateway] + mock_base64_url_encode.return_value = 'gateway_uid' + + PAMGatewayRemoveCommand().execute(params, gateway='gateway_uid') + + self.assertTrue(mock_remove_gateway.called) + + +# --- Legacy discoveryrotation_v1 module (reachable via `pam legacy`) --- +# Same enforcement gate, imported from discoveryrotation and reused, but +# exercised against the v1 command classes to guard the legacy bypass path. + +class TestLegacyPAMCreateGatewayCommandEnforcement(unittest.TestCase): + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.create_gateway') + def test_denied_when_enforcement_absent(self, mock_create_gateway): + params = _params_with_enforcement('denied_absent') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + result = discoveryrotation_v1.PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertIsNone(result) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.create_gateway') + def test_denied_when_enforcement_explicit_false(self, mock_create_gateway): + params = _params_with_enforcement('denied_explicit_false') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + result = discoveryrotation_v1.PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertIsNone(result) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.create_gateway') + def test_allowed_when_enforcement_true(self, mock_create_gateway): + params = _params_with_enforcement('allowed') + mock_create_gateway.return_value = 'US:one-time-token' + + discoveryrotation_v1.PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application', return_value=True) + + self.assertTrue(mock_create_gateway.called) + + +class TestLegacyPAMGatewayRemoveCommandEnforcement(unittest.TestCase): + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.get_all_gateways') + def test_denied_when_enforcement_absent(self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('denied_absent') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + discoveryrotation_v1.PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.get_all_gateways') + def test_denied_when_enforcement_explicit_false(self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('denied_explicit_false') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + discoveryrotation_v1.PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation_v1.utils.base64_url_encode') + def test_allowed_when_enforcement_true( + self, mock_base64_url_encode, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('allowed') + gateway = MagicMock(controllerUid=b'controller_uid', controllerName='pocbyuser') + mock_get_all_gateways.return_value = [gateway] + mock_base64_url_encode.return_value = 'gateway_uid' + + discoveryrotation_v1.PAMGatewayRemoveCommand().execute(params, gateway='gateway_uid') + + self.assertTrue(mock_remove_gateway.called) + + +class TestMixedRolePermissions(unittest.TestCase): + """Partial-permission / mixed-role coverage: a role holding many other + PAM permissions must still be gated on allow_pam_gateway specifically, + neither over-denying (other permissions present) nor over-allowing + (mistaking "booleans list is non-empty" for "gateway is allowed").""" + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_create_denied_with_many_other_permissions_granted( + self, mock_get_app_record, mock_create_gateway): + params = _params_with_mixed_role(gateway_allowed=False) + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertFalse(mock_get_app_record.called) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_ksm_app_display_info') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_create_allowed_with_many_other_permissions_granted( + self, mock_get_app_record, mock_get_ksm_app_display_info, mock_create_gateway): + params = _params_with_mixed_role(gateway_allowed=True) + mock_get_app_record.return_value = {'record_uid': 'app_uid'} + mock_get_ksm_app_display_info.return_value = ('PAM_application', True, 'PAM_application (app_uid)') + mock_create_gateway.return_value = 'US:one-time-token' + + PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application', return_value=True) + + self.assertTrue(mock_create_gateway.called) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + def test_remove_denied_with_many_other_permissions_granted( + self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_mixed_role(gateway_allowed=False) + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/pam/test_pam_nsf_folder_batch.py b/unit-tests/pam/test_pam_nsf_folder_batch.py new file mode 100644 index 000000000..ba15e7d84 --- /dev/null +++ b/unit-tests/pam/test_pam_nsf_folder_batch.py @@ -0,0 +1,163 @@ +"""Tests for batched NSF folder creation used by PAM CyberArk import.""" + +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + +from keepercommander.commands.pam_import.nsf_helpers import create_nsf_folders_batch +from keepercommander.error import CommandError + + +class TestCreateNsfFoldersBatch(unittest.TestCase): + + def _params(self): + params = MagicMock() + params.nested_share_folders = {} + params.subfolder_cache = {} + params.folder_cache = {} + params.environment_variables = {} + return params + + @patch('keepercommander.commands.pam_import.nsf_helpers.sync_down_preserving_nsf_keys') + @patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3') + def test_batches_and_seeds_cache(self, mock_batch, mock_sync): + params = self._params() + mock_batch.return_value = [ + { + 'folder_uid': 'uid-a', + 'folder_key_unencrypted': b'key-a', + 'name': 'SafeA', + 'success': True, + 'message': '', + }, + { + 'folder_uid': 'uid-b', + 'folder_key_unencrypted': b'key-b', + 'name': 'SafeB', + 'success': True, + 'message': '', + }, + ] + + specs = [ + {'name': 'SafeA', 'parent_uid': 'proj'}, + {'name': 'SafeB', 'parent_uid': 'proj'}, + ] + results = create_nsf_folders_batch(params, specs, sync=True, command='pam') + + self.assertEqual(len(results), 2) + mock_batch.assert_called_once_with(params, specs) + mock_sync.assert_called_once_with(params) + self.assertEqual(params.nested_share_folders['uid-a']['name'], 'SafeA') + self.assertEqual(params.nested_share_folders['uid-b']['parent_uid'], 'proj') + + @patch('keepercommander.commands.pam_import.nsf_helpers.sync_down_preserving_nsf_keys') + @patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3') + def test_skips_sync_when_requested(self, mock_batch, mock_sync): + params = self._params() + mock_batch.return_value = [{ + 'folder_uid': 'uid-a', + 'folder_key_unencrypted': b'k', + 'name': 'Config', + 'success': True, + 'message': '', + }] + + create_nsf_folders_batch( + params, [{'name': 'Config', 'parent_uid': 'proj'}], + sync=False, command='pam', + ) + mock_sync.assert_not_called() + + @patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3') + def test_raises_on_failed_folder(self, mock_batch): + params = self._params() + mock_batch.return_value = [{ + 'folder_uid': 'uid-a', + 'success': False, + 'message': 'denied', + 'name': 'Bad', + }] + + with self.assertRaises(CommandError): + create_nsf_folders_batch( + params, [{'name': 'Bad', 'parent_uid': 'proj'}], + sync=False, command='pam', + ) + + @patch('keepercommander.commands.pam_import.nsf_helpers.sync_down_preserving_nsf_keys') + @patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3') + def test_chunks_over_100(self, mock_batch, mock_sync): + params = self._params() + + def _side_effect(_params, chunk): + return [{ + 'folder_uid': f'uid-{i}', + 'folder_key_unencrypted': b'k', + 'name': s['name'], + 'success': True, + 'message': '', + } for i, s in enumerate(chunk)] + + mock_batch.side_effect = _side_effect + specs = [{'name': f'F{i}', 'parent_uid': 'proj'} for i in range(105)] + results = create_nsf_folders_batch(params, specs, sync=True, command='pam') + + self.assertEqual(len(results), 105) + self.assertEqual(mock_batch.call_count, 2) + self.assertEqual(len(mock_batch.call_args_list[0][0][1]), 100) + self.assertEqual(len(mock_batch.call_args_list[1][0][1]), 5) + + +class TestCreateSafeFoldersNsfBatch(unittest.TestCase): + + @patch('keepercommander.commands.pam_import.edit.PAMProjectImportCommand.add_folder_permissions') + @patch('keepercommander.commands.pam_import.nsf_helpers.create_nsf_folders_batch') + def test_two_layer_batch_layout(self, mock_batch, _mock_perms): + from keepercommander.commands.pam_import.edit import PAMProjectImportCommand + + cmd = PAMProjectImportCommand() + params = MagicMock() + res = {"project_folder": "CyberArk Migration", "safe_folders": []} + safe_folder_map = {} + records = [ + {"name": "Win_Local", "safe_name": "Win_Local", "fperm": {}, "uperm": []}, + {"name": "Linux", "safe_name": "Linux", "fperm": {}, "uperm": [{"name": "u@x.com"}]}, + ] + + # Layer1: Config + 2 safes; Layer2: 4 children + def _batch(_params, specs, sync=True, command='pam'): + return [{ + 'folder_uid': f'uid-{spec["name"]}', + 'name': spec['name'], + 'success': True, + } for spec in specs] + + mock_batch.side_effect = _batch + + cmd._create_safe_folders_nsf_batch( + params, 'proj-uid', res, records, + 'CyberArk Migration - Config', safe_folder_map, + ) + + self.assertEqual(mock_batch.call_count, 2) + layer1_specs = mock_batch.call_args_list[0][0][1] + layer2_specs = mock_batch.call_args_list[1][0][1] + self.assertEqual( + [s['name'] for s in layer1_specs], + ['CyberArk Migration - Config', 'Win_Local', 'Linux'], + ) + self.assertEqual(len(layer2_specs), 4) + self.assertEqual(res['config_folder_uid'], 'uid-CyberArk Migration - Config') + self.assertEqual(safe_folder_map['Win_Local'], 'uid-Win_Local') + self.assertIn('Win_Local/Win_Local - Resources', safe_folder_map) + self.assertIn('Linux/Linux - Users', safe_folder_map) + self.assertEqual(len(res['safe_folders']), 2) + _mock_perms.assert_called_once() + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/pam/test_pam_project_import_nsf.py b/unit-tests/pam/test_pam_project_import_nsf.py index 02a6a6be1..0ace49947 100644 --- a/unit-tests/pam/test_pam_project_import_nsf.py +++ b/unit-tests/pam/test_pam_project_import_nsf.py @@ -95,7 +95,10 @@ def test_import_record_objects_use_nsf_aware_record_add_helper(): assert uid == 'record_uid' add_record.assert_called_once() assert add_record.call_args.args[2] == 'root_nsf' - assert add_record.call_args.kwargs == {'command': 'pam-project-import'} + assert add_record.call_args.kwargs == { + 'command': 'pam-project-import', + 'sync_after': False, + } ROOT_NAME = PAMProjectImportCommand.PAM_ROOT_FOLDER_NAME @@ -244,22 +247,21 @@ def test_process_folders_uses_existing_nsf_root_and_creates_nsf_children(): } created = [] - def create_folder(params_arg, folder_name, parent_uid=None): - uid = f'nsf_{len(created) + 1}' - created.append((folder_name, parent_uid, uid)) - params_arg.nested_share_folders[uid] = { - 'name': folder_name, - 'parent_uid': parent_uid, - 'folder_key_unencrypted': b'k' * 32, - } - return { - 'success': True, - 'folder_uid': uid, - 'folder_key_unencrypted': b'k' * 32, - } - - with patch('keepercommander.nested_share_folder.folder_api.create_folder_v3', - side_effect=create_folder) as create_folder_v3, \ + def create_folders_batch(_params, folder_specs): + results = [] + for spec in folder_specs: + uid = f'nsf_{len(created) + 1}' + created.append((spec['name'], spec.get('parent_uid'), uid)) + results.append({ + 'success': True, + 'folder_uid': uid, + 'folder_key_unencrypted': b'k' * 32, + 'name': spec['name'], + }) + return results + + with patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3', + side_effect=create_folders_batch) as create_batch, \ patch('keepercommander.commands.pam_import.nsf_helpers.api.sync_down'), \ patch('keepercommander.commands.pam_import.edit.api.sync_down'): result = PAMProjectImportCommand().process_folders(params, project) @@ -268,7 +270,7 @@ def create_folder(params_arg, folder_name, parent_uid=None): assert result['project_folder_uid'] == 'nsf_1' assert result['resources_folder_uid'] == 'nsf_2' assert result['users_folder_uid'] == 'nsf_3' - assert create_folder_v3.call_count == 3 + assert create_batch.call_count == 3 assert created == [ ('Project 1', 'root_nsf', 'nsf_1'), ('Project 1 - Resources', 'nsf_1', 'nsf_2'), @@ -298,22 +300,21 @@ def test_process_folders_with_nsf_flag_ignores_legacy_root_folder(): } created = [] - def create_folder(params_arg, folder_name, parent_uid=None): - uid = f'nsf_{len(created) + 1}' - created.append((folder_name, parent_uid, uid)) - params_arg.nested_share_folders[uid] = { - 'name': folder_name, - 'parent_uid': parent_uid, - 'folder_key_unencrypted': b'k' * 32, - } - return { - 'success': True, - 'folder_uid': uid, - 'folder_key_unencrypted': b'k' * 32, - } - - with patch('keepercommander.nested_share_folder.folder_api.create_folder_v3', - side_effect=create_folder), \ + def create_folders_batch(_params, folder_specs): + results = [] + for spec in folder_specs: + uid = f'nsf_{len(created) + 1}' + created.append((spec['name'], spec.get('parent_uid'), uid)) + results.append({ + 'success': True, + 'folder_uid': uid, + 'folder_key_unencrypted': b'k' * 32, + 'name': spec['name'], + }) + return results + + with patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3', + side_effect=create_folders_batch), \ patch('keepercommander.commands.pam_import.nsf_helpers.api.sync_down'), \ patch('keepercommander.commands.pam_import.edit.api.sync_down'): result = PAMProjectImportCommand().process_folders(params, project) @@ -354,18 +355,19 @@ def test_create_subfolder_seeds_folder_key_and_survives_sync_wipe(): params = _params() folder_key = b'f' * 32 - def create_folder(_params, folder_name, parent_uid=None): - return { + def create_folders_batch(_params, folder_specs): + return [{ 'success': True, 'folder_uid': 'new_nsf', 'folder_key_unencrypted': folder_key, - } + 'name': folder_specs[0]['name'], + }] def wipe_nsf(_params): _params.nested_share_folders.clear() - with patch('keepercommander.nested_share_folder.folder_api.create_folder_v3', - side_effect=create_folder), \ + with patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3', + side_effect=create_folders_batch), \ patch('keepercommander.commands.pam_import.nsf_helpers.api.sync_down', side_effect=wipe_nsf): uid = PAMProjectImportCommand().create_subfolder( diff --git a/unit-tests/pam/test_pam_rotation.py b/unit-tests/pam/test_pam_rotation.py index 79f723e88..96a5c6b2c 100644 --- a/unit-tests/pam/test_pam_rotation.py +++ b/unit-tests/pam/test_pam_rotation.py @@ -198,6 +198,117 @@ def test_execute_with_valid_record(self, mock_TunnelDAG, mock_load): self.assertTrue(mock_TunnelDAG.called) self.assertEqual(mock_typed_record.record_key, b'\x00' * 16) + @patch('keepercommander.commands.discoveryrotation.api.sync_down') + @patch('keepercommander.vault.KeeperRecord.load') + @patch('keepercommander.commands.discoveryrotation.TunnelDAG') + @patch('keepercommander.rest_api.SERVER_PUBLIC_KEYS', {8: ec.generate_private_key(ec.SECP256R1()).public_key()}) + def test_execute_default_trust_cache_skips_forced_sync(self, mock_TunnelDAG, mock_load, mock_sync_down): + # Direct .execute() calls (bulk PAM import/extend) default to trust_cache=True + # and must not pay for an extra sync_down per record. + mock_params, mock_typed_record = create_mock_params_and_record() + mock_load.return_value = mock_typed_record + + mock_dag_instance = mock_TunnelDAG.return_value + mock_dag_instance.linking_dag.has_graph = True + mock_dag_instance.check_if_resource_has_admin.return_value = True + mock_dag_instance.get_all_owners.return_value = [TEST_RESOURCE_UID] + mock_dag_instance.resource_belongs_to_config.return_value = True + mock_dag_instance.user_belongs_to_resource.return_value = True + mock_dag_instance.record.record_uid = TEST_CONFIG_UID + + self.command.execute(mock_params, record_name=TEST_RECORD_UID, force=True) + mock_sync_down.assert_not_called() + + @patch('keepercommander.commands.discoveryrotation.api.sync_down') + @patch('keepercommander.vault.KeeperRecord.load') + @patch('keepercommander.commands.discoveryrotation.TunnelDAG') + @patch('keepercommander.rest_api.SERVER_PUBLIC_KEYS', {8: ec.generate_private_key(ec.SECP256R1()).public_key()}) + def test_execute_trust_cache_false_forces_sync_down(self, mock_TunnelDAG, mock_load, mock_sync_down): + # trust_cache=False (the mode used by CLI dispatch) must refresh the local + # rotation cache before any cached revision is read, to avoid submitting a + # stale revision that collides with a real change made elsewhere. + mock_params, mock_typed_record = create_mock_params_and_record() + mock_load.return_value = mock_typed_record + + mock_dag_instance = mock_TunnelDAG.return_value + mock_dag_instance.linking_dag.has_graph = True + mock_dag_instance.check_if_resource_has_admin.return_value = True + mock_dag_instance.get_all_owners.return_value = [TEST_RESOURCE_UID] + mock_dag_instance.resource_belongs_to_config.return_value = True + mock_dag_instance.user_belongs_to_resource.return_value = True + mock_dag_instance.record.record_uid = TEST_CONFIG_UID + + self.command.execute(mock_params, record_name=TEST_RECORD_UID, force=True, trust_cache=False) + mock_sync_down.assert_called_once_with(mock_params) + + @patch('keepercommander.commands.discoveryrotation.api.sync_down') + @patch('keepercommander.vault.KeeperRecord.load') + @patch('keepercommander.commands.discoveryrotation.TunnelDAG') + @patch('keepercommander.rest_api.SERVER_PUBLIC_KEYS', {8: ec.generate_private_key(ec.SECP256R1()).public_key()}) + def test_execute_trust_cache_non_false_values_skip_forced_sync(self, mock_TunnelDAG, mock_load, mock_sync_down): + # Only a literal False must opt into the forced-resync path; any other + # value (None, 0, garbage) falls back to the safe/performant True default. + mock_params, mock_typed_record = create_mock_params_and_record() + mock_load.return_value = mock_typed_record + + mock_dag_instance = mock_TunnelDAG.return_value + mock_dag_instance.linking_dag.has_graph = True + mock_dag_instance.check_if_resource_has_admin.return_value = True + mock_dag_instance.get_all_owners.return_value = [TEST_RESOURCE_UID] + mock_dag_instance.resource_belongs_to_config.return_value = True + mock_dag_instance.user_belongs_to_resource.return_value = True + mock_dag_instance.record.record_uid = TEST_CONFIG_UID + + for ambiguous_value in (None, 0, ''): + mock_sync_down.reset_mock() + self.command.execute(mock_params, record_name=TEST_RECORD_UID, force=True, + trust_cache=ambiguous_value) + mock_sync_down.assert_not_called() + + def test_execute_args_defaults_trust_cache_false_and_sets_sync_data(self): + mock_params = MagicMock() + mock_params.sync_data = False + with patch.object(self.command, 'execute') as mock_execute: + self.command.execute_args(mock_params, f'--record {TEST_RECORD_UID} --force') + self.assertTrue(mock_execute.called) + called_kwargs = mock_execute.call_args.kwargs + self.assertEqual(called_kwargs.get('trust_cache'), False) + self.assertTrue(mock_params.sync_data) + + def test_execute_args_trust_cache_true_override_skips_sync_data(self): + # A caller could still opt a dispatched invocation into cache-trusting mode + # explicitly; when it does, execute_args must not force a post-sync. + mock_params = MagicMock() + mock_params.sync_data = False + with patch.object(self.command, 'execute') as mock_execute: + self.command.execute_args(mock_params, f'--record {TEST_RECORD_UID} --force', trust_cache=True) + called_kwargs = mock_execute.call_args.kwargs + self.assertEqual(called_kwargs.get('trust_cache'), True) + self.assertFalse(mock_params.sync_data) + + def test_execute_args_ambiguous_trust_cache_values_normalize_to_true(self): + # Same identity rule at the execute_args() layer: only a literal False + # forces the post-sync; None/0/'' normalize to True (no forced post-sync). + for ambiguous_value in (None, 0, ''): + mock_params = MagicMock() + mock_params.sync_data = False + with patch.object(self.command, 'execute') as mock_execute: + self.command.execute_args(mock_params, f'--record {TEST_RECORD_UID} --force', + trust_cache=ambiguous_value) + called_kwargs = mock_execute.call_args.kwargs + self.assertEqual(called_kwargs.get('trust_cache'), True) + self.assertFalse(mock_params.sync_data) + + def test_execute_args_sets_sync_data_even_when_execute_raises(self): + # The post-sync guarantee must hold even if the command fails validation + # partway through (e.g. "No PAM record is found"). + mock_params = MagicMock() + mock_params.sync_data = False + with patch.object(self.command, 'execute', side_effect=CommandError('', 'boom')): + with self.assertRaises(CommandError): + self.command.execute_args(mock_params, f'--record {TEST_RECORD_UID} --force') + self.assertTrue(mock_params.sync_data) + @patch('keepercommander.commands.discoveryrotation.api.sync_down') @patch('keepercommander.commands.discoveryrotation.router_set_record_rotation_information') @patch('keepercommander.vault_extensions.find_records') @@ -860,6 +971,195 @@ def test_table_mode_returns_none(self, mock_rrg, mock_schedules): result = cmd.execute(mock_params, record_uid=record_uid, format='table') self.assertIsNone(result) + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation.router_get_rotation_schedules') + @patch('keepercommander.commands.discoveryrotation.record_rotation_get') + def test_gateway_name_resolved_from_uid_when_empty(self, mock_rrg, mock_schedules, mock_get_gateways): + """When controllerName is empty, it should be resolved from gateway list using controllerUid.""" + from keeper_secrets_manager_core.utils import url_safe_str_to_bytes + record_uid = 'test_record_uid_' + record_uid_bytes = url_safe_str_to_bytes(record_uid) + + rri = self._make_rri('RRS_ONLINE') + rri.controllerName = '' + + mock_rrg.return_value = rri + + sched_mock = MagicMock() + sched_mock.schedules = [self._make_schedule(record_uid_bytes)] + mock_schedules.return_value = sched_mock + + mock_gateway = MagicMock() + mock_gateway.controllerUid = rri.controllerUid + mock_gateway.controllerName = 'gw-test-resolved' + mock_get_gateways.return_value = [mock_gateway] + + mock_params = create_mock_params() + mock_params.record_cache = {} + + cmd = PAMRouterGetRotationInfo() + result = cmd.execute(mock_params, record_uid=record_uid, format='json') + + self.assertIsNotNone(result, "Expected JSON string, got None") + data = json.loads(result) + self.assertEqual(data['gateway_name'], 'gw-test-resolved') + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation.router_get_rotation_schedules') + @patch('keepercommander.commands.discoveryrotation.record_rotation_get') + def test_gateway_name_present_unchanged(self, mock_rrg, mock_schedules, mock_get_gateways): + """When controllerName is present, it should be used without looking up gateways.""" + from keeper_secrets_manager_core.utils import url_safe_str_to_bytes + record_uid = 'test_record_uid_' + record_uid_bytes = url_safe_str_to_bytes(record_uid) + + rri = self._make_rri('RRS_ONLINE') + rri.controllerName = 'gw-original' + + mock_rrg.return_value = rri + + sched_mock = MagicMock() + sched_mock.schedules = [self._make_schedule(record_uid_bytes)] + mock_schedules.return_value = sched_mock + + mock_params = create_mock_params() + mock_params.record_cache = {} + + cmd = PAMRouterGetRotationInfo() + result = cmd.execute(mock_params, record_uid=record_uid, format='json') + + self.assertIsNotNone(result) + data = json.loads(result) + self.assertEqual(data['gateway_name'], 'gw-original') + mock_get_gateways.assert_not_called() + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation.router_get_rotation_schedules') + @patch('keepercommander.commands.discoveryrotation.record_rotation_get') + def test_gateway_name_empty_no_match_falls_back_to_dash(self, mock_rrg, mock_schedules, mock_get_gateways): + """When controllerName is empty and no gateway matches, should fall back to '-'.""" + from keeper_secrets_manager_core.utils import url_safe_str_to_bytes + record_uid = 'test_record_uid_' + record_uid_bytes = url_safe_str_to_bytes(record_uid) + + rri = self._make_rri('RRS_ONLINE') + rri.controllerName = '' + + mock_rrg.return_value = rri + + sched_mock = MagicMock() + sched_mock.schedules = [self._make_schedule(record_uid_bytes)] + mock_schedules.return_value = sched_mock + + mock_gateway = MagicMock() + mock_gateway.controllerUid = b'different_uid_' + mock_gateway.controllerName = 'gw-other' + mock_get_gateways.return_value = [mock_gateway] + + mock_params = create_mock_params() + mock_params.record_cache = {} + + cmd = PAMRouterGetRotationInfo() + result = cmd.execute(mock_params, record_uid=record_uid, format='json') + + self.assertIsNotNone(result) + data = json.loads(result) + self.assertEqual(data['gateway_name'], '-') + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation.router_get_rotation_schedules') + @patch('keepercommander.commands.discoveryrotation.record_rotation_get') + def test_gateway_name_resolved_with_different_uid_types(self, mock_rrg, mock_schedules, mock_get_gateways): + """When controllerUid types differ (bytes vs string), should still resolve correctly.""" + from keeper_secrets_manager_core.utils import url_safe_str_to_bytes + from keepercommander import utils + record_uid = 'test_record_uid_' + record_uid_bytes = url_safe_str_to_bytes(record_uid) + + rri = self._make_rri('RRS_ONLINE') + rri.controllerName = '' + + mock_rrg.return_value = rri + + sched_mock = MagicMock() + sched_mock.schedules = [self._make_schedule(record_uid_bytes)] + mock_schedules.return_value = sched_mock + + mock_gateway = MagicMock() + mock_gateway.controllerUid = utils.base64_url_encode(rri.controllerUid) + mock_gateway.controllerName = 'gw-test-resolved' + mock_get_gateways.return_value = [mock_gateway] + + mock_params = create_mock_params() + mock_params.record_cache = {} + + cmd = PAMRouterGetRotationInfo() + result = cmd.execute(mock_params, record_uid=record_uid, format='json') + + self.assertIsNotNone(result) + data = json.loads(result) + self.assertEqual(data['gateway_name'], 'gw-test-resolved') + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation.router_get_rotation_schedules') + @patch('keepercommander.commands.discoveryrotation.record_rotation_get') + def test_gateway_list_returns_none_falls_back_gracefully(self, mock_rrg, mock_schedules, mock_get_gateways): + """When get_all_gateways returns None, should fall back to '-' gracefully.""" + from keeper_secrets_manager_core.utils import url_safe_str_to_bytes + record_uid = 'test_record_uid_' + record_uid_bytes = url_safe_str_to_bytes(record_uid) + + rri = self._make_rri('RRS_ONLINE') + rri.controllerName = '' + + mock_rrg.return_value = rri + + sched_mock = MagicMock() + sched_mock.schedules = [self._make_schedule(record_uid_bytes)] + mock_schedules.return_value = sched_mock + + mock_get_gateways.return_value = None + + mock_params = create_mock_params() + mock_params.record_cache = {} + + cmd = PAMRouterGetRotationInfo() + result = cmd.execute(mock_params, record_uid=record_uid, format='json') + + self.assertIsNotNone(result) + data = json.loads(result) + self.assertEqual(data['gateway_name'], '-') + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation.router_get_rotation_schedules') + @patch('keepercommander.commands.discoveryrotation.record_rotation_get') + def test_gateway_list_raises_exception_falls_back_gracefully(self, mock_rrg, mock_schedules, mock_get_gateways): + """When get_all_gateways raises an exception, should fall back to '-' without crashing.""" + from keeper_secrets_manager_core.utils import url_safe_str_to_bytes + record_uid = 'test_record_uid_' + record_uid_bytes = url_safe_str_to_bytes(record_uid) + + rri = self._make_rri('RRS_ONLINE') + rri.controllerName = '' + + mock_rrg.return_value = rri + + sched_mock = MagicMock() + sched_mock.schedules = [self._make_schedule(record_uid_bytes)] + mock_schedules.return_value = sched_mock + + mock_get_gateways.side_effect = RuntimeError("Gateway service unavailable") + + mock_params = create_mock_params() + mock_params.record_cache = {} + + cmd = PAMRouterGetRotationInfo() + result = cmd.execute(mock_params, record_uid=record_uid, format='json') + + self.assertIsNotNone(result) + data = json.loads(result) + self.assertEqual(data['gateway_name'], '-') + class TestUsesDefaultRotationSchedule(unittest.TestCase): diff --git a/unit-tests/pam/test_pam_tunnel.py b/unit-tests/pam/test_pam_tunnel.py index 41c55e7f8..0bfc140c0 100644 --- a/unit-tests/pam/test_pam_tunnel.py +++ b/unit-tests/pam/test_pam_tunnel.py @@ -2,6 +2,7 @@ from unittest import mock from keepercommander.error import CommandError +from keepercommander.commands.tunnel_and_connections import PAMTunnelDiagnoseCommand import datetime import socket @@ -154,3 +155,83 @@ def test_uniqueness(self): random_bytes1 = generate_random_bytes() random_bytes2 = generate_random_bytes() self.assertNotEqual(random_bytes1, random_bytes2) + + +class TestPAMTunnelDiagnose(unittest.TestCase): + def test_execute_passes_session_proxy_to_https_probes(self): + proxies = {'http': 'http://proxy.example:8080', 'https': 'http://proxy.example:8080'} + params = mock.MagicMock() + params.server = 'keepersecurity.com' + params.rest_context.proxies = proxies + params.ssl_verify = '/path/to/ca.pem' + + with mock.patch('keepercommander.commands.tunnel_and_connections.get_relay_host', + return_value='relay.example'), \ + mock.patch('keepercommander.commands.tunnel_and_connections.get_router_host', + return_value='router.example'), \ + mock.patch('keepercommander.commands.tunnel_and_connections.get_or_create_tube_registry', + return_value=None), \ + mock.patch('keepercommander.commands.tunnel_and_connections.socket.getaddrinfo', + return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('203.0.113.1', 0))]), \ + mock.patch('keepercommander.commands.tunnel_and_connections.socket.gethostbyname', + return_value='203.0.113.1'), \ + mock.patch.object(PAMTunnelDiagnoseCommand, '_test_https', + return_value=(True, 'reachable', 1)) as test_https, \ + mock.patch.object(PAMTunnelDiagnoseCommand, '_test_websocket', + return_value=(True, 'reachable', 1)) as test_websocket, \ + mock.patch.object(PAMTunnelDiagnoseCommand, '_test_tcp_stun', + return_value=(True, 'reachable', 1, None)), \ + mock.patch.object(PAMTunnelDiagnoseCommand, '_test_udp_stun', + return_value=(True, 'reachable', 1, None)), \ + mock.patch.object(PAMTunnelDiagnoseCommand, '_test_turn', + return_value=(True, 'reachable', 1)), \ + mock.patch.object(PAMTunnelDiagnoseCommand, '_test_udp_port', + return_value=(True, 1)): + PAMTunnelDiagnoseCommand().execute(params) + + test_https.assert_called_once_with( + 'keepersecurity.com', proxies=proxies, verify='/path/to/ca.pem') + test_websocket.assert_called_once_with( + 'router.example', proxies=proxies, verify='/path/to/ca.pem') + + def test_https_uses_configured_proxy(self): + proxies = {'http': 'http://proxy.example:8080', 'https': 'http://proxy.example:8080'} + with mock.patch('keepercommander.commands.tunnel_and_connections.requests.get') as mock_get: + mock_get.return_value.status_code = 200 + + passed, _, _ = PAMTunnelDiagnoseCommand._test_https( + 'api.example', proxies=proxies, verify='/path/to/ca.pem') + + self.assertTrue(passed) + mock_get.assert_called_once_with( + 'https://api.example:443/', + headers={'User-Agent': 'keeper-pam-diagnose/1.0'}, + proxies=proxies, + verify='/path/to/ca.pem', + timeout=10, + stream=True, + ) + + def test_websocket_uses_configured_proxy(self): + proxies = {'http': 'http://proxy.example:8080', 'https': 'http://proxy.example:8080'} + with mock.patch('keepercommander.commands.tunnel_and_connections.requests.get') as mock_get: + mock_get.return_value.status_code = 101 + + passed, _, _ = PAMTunnelDiagnoseCommand._test_websocket( + 'router.example', proxies=proxies, verify='/path/to/ca.pem') + + self.assertTrue(passed) + mock_get.assert_called_once_with( + 'https://router.example:443/', + headers={ + 'Upgrade': 'websocket', + 'Connection': 'Upgrade', + 'Sec-WebSocket-Key': 'dGhlIHNhbXBsZSBub25jZQ==', + 'Sec-WebSocket-Version': '13', + 'User-Agent': 'keeper-pam-diagnose/1.0', + }, + proxies=proxies, + verify='/path/to/ca.pem', + timeout=10, + stream=True, + ) diff --git a/unit-tests/pam/test_pam_tunnel_ephemeral_credential.py b/unit-tests/pam/test_pam_tunnel_ephemeral_credential.py new file mode 100644 index 000000000..eb13686f6 --- /dev/null +++ b/unit-tests/pam/test_pam_tunnel_ephemeral_credential.py @@ -0,0 +1,316 @@ +""" +Unit tests for the KC (Commander) half of the ephemeral/--credential proxy +tunnel plan: + +- Section 2.1: launch-credential preflight falls back to ephemeral JIT. +- Section 2.2: gateway timeout bump for ephemeral tunnel starts. +- Section 3.2: --credential/-cr proxy-target classification, allowSupplyUser + gate, and record resolution for `pam tunnel start`. + +These test the small, pure/near-pure functions extracted from +tunnel_and_connections.py / tunnel_helpers.py rather than the full CLI +command (which would require mocking KeeperParams/TunnelDAG/gateway network +calls not otherwise exercised by this repo's test suite). +""" + +import unittest +from unittest import mock + +from keepercommander.commands.tunnel_and_connections import ( + _classify_proxy_target, + _detect_proxy_backend, + _pam_allow_supply_user, + _resolve_credential_record, + _resource_has_ephemeral_jit, +) +from keepercommander.commands.tunnel.port_forward.tunnel_helpers import ( + credential_override_data_fields, + credential_override_input_fields, + ephemeral_gateway_timeout_ms, +) + + +class TestClassifyProxyTarget(unittest.TestCase): + def test_database_record_is_keeperdb_proxy(self): + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamDatabase', '', True, None) + self.assertTrue(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + self.assertIsNone(error) + + def test_machine_rdp_record_is_keeperrdp_proxy(self): + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamMachine', 'rdp', True, None) + self.assertFalse(is_db) + self.assertTrue(is_rdp) + self.assertFalse(is_ssh) + self.assertIsNone(error) + + def test_machine_ssh_record_is_keeperssh_proxy(self): + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamMachine', 'ssh', True, None) + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertTrue(is_ssh) + self.assertIsNone(error) + + def test_credential_alone_detects_ssh_proxy_without_proxy_flag(self): + # --credential alone (no --proxy) still resolves to KeeperSSH Proxy + # when the record is a pamMachine+SSH record — the gateway auto-routes + # from the record's own settings regardless of the client-side flag. + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamMachine', 'ssh', False, 'some-record') + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertTrue(is_ssh) + self.assertIsNone(error) + + def test_unsupported_type_with_proxy_flag_gives_proxy_message(self): + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamMachine', 'vnc', True, None) + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + self.assertIn('--proxy is supported on', error) + + def test_unsupported_type_with_credential_only_gives_plain_tunnel_message(self): + # --credential with no --proxy, on a record that isn't proxy-capable + # at all: distinct message from the --proxy-driven one above. + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamMachine', 'vnc', False, 'some-record') + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + self.assertEqual(error, "--credential requires a proxied tunnel; plain tunnels don't authenticate.") + + def test_credential_on_database_record_is_rejected(self): + # KeeperDB Proxy keeps "ephemeral always wins, no override" — reject + # --credential outright instead of accepting a no-op flag. + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamDatabase', '', True, 'some-record') + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + self.assertEqual(error, '--credential is not yet supported for KeeperDB Proxy tunnels.') + + def test_credential_on_database_record_rejected_even_without_proxy_flag(self): + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamDatabase', '', False, 'some-record') + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + self.assertEqual(error, '--credential is not yet supported for KeeperDB Proxy tunnels.') + + def test_credential_alone_detects_rdp_proxy_without_proxy_flag(self): + # --credential alone (no --proxy) still resolves to KeeperRDP Proxy + # when the record is a pamMachine+RDP record — the gateway auto-routes + # from the record's own settings regardless of the client-side flag. + is_db, is_rdp, is_ssh, error = _classify_proxy_target('rec-uid', 'pamMachine', 'rdp', False, 'some-record') + self.assertFalse(is_db) + self.assertTrue(is_rdp) + self.assertFalse(is_ssh) + self.assertIsNone(error) + + +class TestDetectProxyBackend(unittest.TestCase): + """_detect_proxy_backend drives the post-start proxy-ready banner for + *every* tunnel start on a proxy-enabled record, not just ones started + with --proxy/--credential — so it must read the record's actual + allowKeeper*Proxy flags directly, unlike _classify_proxy_target (which + only cares about record type/protocol, not whether the flag is set). + """ + + def test_database_proxy_enabled_via_portforward_bucket(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend( + 'pamDatabase', '', {'portForward': {'allowKeeperDBProxy': True}}) + self.assertTrue(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + + def test_database_proxy_enabled_via_legacy_connection_bucket(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend( + 'pamDatabase', '', {'connection': {'allowKeeperDBProxy': True}}) + self.assertTrue(is_db) + + def test_database_record_without_flag_is_not_a_proxy(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend('pamDatabase', '', {'connection': {}}) + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + + def test_rdp_proxy_enabled(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend( + 'pamMachine', 'rdp', {'portForward': {'allowKeeperRDPProxy': True}}) + self.assertFalse(is_db) + self.assertTrue(is_rdp) + self.assertFalse(is_ssh) + + def test_rdp_record_without_flag_is_not_a_proxy(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend('pamMachine', 'rdp', {'portForward': {}}) + self.assertFalse(is_rdp) + + def test_ssh_proxy_enabled(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend( + 'pamMachine', 'ssh', {'portForward': {'allowKeeperSSHProxy': True}}) + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertTrue(is_ssh) + + def test_ssh_record_without_flag_is_not_a_proxy(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend('pamMachine', 'ssh', {'portForward': {}}) + self.assertFalse(is_ssh) + + def test_unrelated_record_type_is_not_a_proxy(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend( + 'pamMachine', 'vnc', {'portForward': {'allowKeeperSSHProxy': True}}) + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + + def test_non_dict_pam_settings_is_not_a_proxy(self): + is_db, is_rdp, is_ssh = _detect_proxy_backend('pamDatabase', '', None) + self.assertFalse(is_db) + self.assertFalse(is_rdp) + self.assertFalse(is_ssh) + + +class TestPamAllowSupplyUser(unittest.TestCase): + def test_true_when_set(self): + self.assertTrue(_pam_allow_supply_user({'connection': {'allowSupplyUser': True}})) + + def test_false_when_absent(self): + self.assertFalse(_pam_allow_supply_user({'connection': {}})) + + def test_false_when_connection_missing(self): + self.assertFalse(_pam_allow_supply_user({})) + + def test_false_when_not_a_dict(self): + self.assertFalse(_pam_allow_supply_user(None)) + + +class TestResourceHasEphemeralJit(unittest.TestCase): + @mock.patch('keepercommander.commands.tunnel_and_connections.get_resource_jit_settings') + def test_true_when_create_ephemeral_set(self, mock_get_jit): + mock_get_jit.return_value = {'createEphemeral': True} + self.assertTrue(_resource_has_ephemeral_jit(object(), 'uid1')) + + @mock.patch('keepercommander.commands.tunnel_and_connections.get_resource_jit_settings') + def test_false_when_create_ephemeral_false(self, mock_get_jit): + mock_get_jit.return_value = {'createEphemeral': False} + self.assertFalse(_resource_has_ephemeral_jit(object(), 'uid1')) + + @mock.patch('keepercommander.commands.tunnel_and_connections.get_resource_jit_settings') + def test_false_when_no_jit_settings(self, mock_get_jit): + mock_get_jit.return_value = None + self.assertFalse(_resource_has_ephemeral_jit(object(), 'uid1')) + + @mock.patch('keepercommander.commands.tunnel_and_connections.get_resource_jit_settings') + def test_false_on_read_error(self, mock_get_jit): + mock_get_jit.side_effect = Exception('DAG unavailable') + self.assertFalse(_resource_has_ephemeral_jit(object(), 'uid1')) + + +class _FakeRecord: + def __init__(self, title): + self.title = title + + +class _FakeParams: + def __init__(self, record_cache): + self.record_cache = record_cache + + +class TestResolveCredentialRecord(unittest.TestCase): + def test_uid_hit_returns_uid_directly(self): + params = _FakeParams({'abc123': {}}) + self.assertEqual(_resolve_credential_record(params, 'abc123'), 'abc123') + + def test_empty_token_returns_none(self): + params = _FakeParams({}) + self.assertIsNone(_resolve_credential_record(params, '')) + self.assertIsNone(_resolve_credential_record(params, None)) + + @mock.patch('keepercommander.commands.tunnel_and_connections.try_resolve_path', return_value=None) + @mock.patch('keepercommander.vault.KeeperRecord.load') + def test_exact_title_match_returns_uid(self, mock_load, mock_resolve_path): + params = _FakeParams({'uid-1': {}, 'uid-2': {}}) + + def load_side_effect(_params, uid): + return {'uid-1': _FakeRecord('My Server'), 'uid-2': _FakeRecord('Other Record')}[uid] + + mock_load.side_effect = load_side_effect + self.assertEqual(_resolve_credential_record(params, 'My Server'), 'uid-1') + + @mock.patch('keepercommander.commands.tunnel_and_connections.try_resolve_path', return_value=None) + @mock.patch('keepercommander.vault.KeeperRecord.load') + def test_ambiguous_title_returns_none(self, mock_load, mock_resolve_path): + params = _FakeParams({'uid-1': {}, 'uid-2': {}}) + mock_load.return_value = _FakeRecord('Duplicate Title') + self.assertIsNone(_resolve_credential_record(params, 'Duplicate Title')) + + @mock.patch('keepercommander.commands.tunnel_and_connections.try_resolve_path', return_value=None) + @mock.patch('keepercommander.vault.KeeperRecord.load', return_value=None) + def test_no_match_returns_none(self, mock_load, mock_resolve_path): + params = _FakeParams({'uid-1': {}}) + self.assertIsNone(_resolve_credential_record(params, 'Nonexistent')) + + +class TestEphemeralGatewayTimeoutMs(unittest.TestCase): + def test_default_when_not_ephemeral(self): + self.assertEqual(ephemeral_gateway_timeout_ms(False, default=30000), 30000) + + def test_bumped_when_ephemeral(self): + self.assertEqual(ephemeral_gateway_timeout_ms(True, default=30000), 120000) + + @mock.patch.dict('os.environ', {'PAM_GATEWAY_OFFER_TIMEOUT_EPHEMERAL_MS': '45000'}) + def test_bump_respects_env_override(self): + self.assertEqual(ephemeral_gateway_timeout_ms(True, default=30000), 45000) + + +class TestCredentialOverridePayloadFields(unittest.TestCase): + def test_data_fields_for_user_supplied(self): + # RDP-style override: username/password only, no key material. + # Regression check that adding SSH's private_key/passphrase support + # doesn't affect RDP's existing username/password-only case. + fields = credential_override_data_fields('userSupplied', {'username': 'bob', 'password': 'secret'}) + self.assertEqual(fields, {'username': 'bob', 'password': 'secret'}) + + def test_data_fields_include_private_key_and_passphrase_when_present(self): + fields = credential_override_data_fields('userSupplied', { + 'username': 'bob', + 'password': '', + 'private_key': '-----BEGIN OPENSSH PRIVATE KEY-----...', + 'passphrase': 'key-passphrase', + }) + self.assertEqual(fields, { + 'username': 'bob', + 'password': '', + 'private_key': '-----BEGIN OPENSSH PRIVATE KEY-----...', + 'passphrase': 'key-passphrase', + }) + + def test_data_fields_omit_private_key_and_passphrase_when_absent(self): + fields = credential_override_data_fields('userSupplied', {'username': 'bob', 'password': 'secret'}) + self.assertNotIn('private_key', fields) + self.assertNotIn('passphrase', fields) + + def test_data_fields_omit_private_key_and_passphrase_when_empty(self): + fields = credential_override_data_fields('userSupplied', { + 'username': 'bob', 'password': 'secret', 'private_key': '', 'passphrase': '', + }) + self.assertNotIn('private_key', fields) + self.assertNotIn('passphrase', fields) + + def test_data_fields_empty_when_not_user_supplied(self): + self.assertEqual(credential_override_data_fields(None, {'username': 'bob', 'password': 'secret'}), {}) + self.assertEqual(credential_override_data_fields('linked', {'username': 'bob', 'password': 'secret'}), {}) + + def test_data_fields_empty_when_no_credential_data(self): + self.assertEqual(credential_override_data_fields('userSupplied', None), {}) + + def test_input_fields_for_user_supplied(self): + self.assertEqual( + credential_override_input_fields('userSupplied'), + {'credentialType': 'userSupplied', 'allowSupplyUser': True}, + ) + + def test_input_fields_empty_when_not_user_supplied(self): + self.assertEqual(credential_override_input_fields(None), {}) + self.assertEqual(credential_override_input_fields('linked'), {}) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/pam/test_proxy_ready_banner.py b/unit-tests/pam/test_proxy_ready_banner.py new file mode 100644 index 000000000..8ba1761be --- /dev/null +++ b/unit-tests/pam/test_proxy_ready_banner.py @@ -0,0 +1,63 @@ +""" +Unit tests for _print_proxy_ready_banner, the KeeperDB/KeeperRDP/KeeperSSH +Proxy "ready" banner printed from TunnelSignalHandler's WebRTC 'connected' +callback (not immediately after start_rust_tunnel returns -- see the +function's docstring for why: a proxy tunnel's offer can be "accepted" well +before the gateway has actually spawned the proxy and authenticated against +the real target, and that can still fail afterward). +""" + +import logging +import unittest + +from keepercommander.commands.tunnel.port_forward.tunnel_helpers import ( + TunnelSession, + _print_proxy_ready_banner, +) + + +def _make_session(proxy_kind, proxy_db_type=None, host='127.0.0.1', port=49152): + return TunnelSession( + tube_id='tube-1', conversation_id='conv-1', gateway_uid='gw-1', symmetric_key=b'key', + host=host, port=port, proxy_kind=proxy_kind, proxy_db_type=proxy_db_type, + ) + + +class TestPrintProxyReadyBanner(unittest.TestCase): + def test_ssh_banner_mentions_listen_and_connect_string(self): + session = _make_session('ssh') + with self.assertLogs(level='INFO') as captured: + _print_proxy_ready_banner(session) + combined = '\n'.join(captured.output) + self.assertIn('KeeperSSH Proxy ready', combined) + self.assertIn('127.0.0.1:49152', combined) + self.assertIn('ssh -p 49152', combined) + + def test_rdp_banner_mentions_mstsc(self): + session = _make_session('rdp') + with self.assertLogs(level='INFO') as captured: + _print_proxy_ready_banner(session) + combined = '\n'.join(captured.output) + self.assertIn('KeeperRDP Proxy ready', combined) + self.assertIn('mstsc', combined) + + def test_db_banner_mysql_specific_connect_string(self): + session = _make_session('db', proxy_db_type='mysql') + with self.assertLogs(level='INFO') as captured: + _print_proxy_ready_banner(session) + combined = '\n'.join(captured.output) + self.assertIn('KeeperDB Proxy ready (mysql)', combined) + self.assertIn('mysql -h 127.0.0.1 -P 49152', combined) + + def test_db_banner_falls_back_to_generic_connect_string_when_type_unknown(self): + session = _make_session('db', proxy_db_type=None) + with self.assertLogs(level='INFO') as captured: + _print_proxy_ready_banner(session) + combined = '\n'.join(captured.output) + self.assertIn('KeeperDB Proxy ready', combined) + self.assertNotIn('KeeperDB Proxy ready (', combined) + self.assertIn('use your database client', combined) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/pam/test_route_message_to_rust_gateway_error.py b/unit-tests/pam/test_route_message_to_rust_gateway_error.py new file mode 100644 index 000000000..c258a62ce --- /dev/null +++ b/unit-tests/pam/test_route_message_to_rust_gateway_error.py @@ -0,0 +1,188 @@ +""" +Regression test: an async Gateway error (is_ok=False) delivered via the +WebSocket message-routing path must close the tube for a KeeperSSH Proxy +tunnel that failed to open, not just stash the error onto fields nobody +reads again. + +Bug: route_message_to_rust()'s is_ok=False branch only set +session.gateway_error_message/gateway_error_event, which are only ever +consumed by the synchronous connect-retry wait in tunnel_and_connections.py. +A KeeperSSH Proxy setup failure arriving after that window closes (Rust's +real upstream auth/connect attempt takes a real network round trip) left +the tube open forever. + +Scope of the fix (deliberately narrow): only the 3 known-fatal KeeperSSH +Proxy setup error codes (auth/timeout/config), only within the same window +as the longest legitimate startup delay (ephemeral JIT account creation, +120s by default). Everything else — the generic proxy error code, unrelated +gateway errors (e.g. 401 on get_leafs), and any error arriving after the +window — must keep today's log-and-stash-only behavior so a plain SSH +tunnel or an already-established, long-running KeeperSSH Proxy tunnel is +never torn down by an unrelated later message. +""" + +import json +import unittest +from unittest import mock + +from keepercommander.commands.tunnel.port_forward.tunnel_helpers import ( + CloseConnectionReasons, + TunnelSession, + ephemeral_gateway_timeout_ms, + get_tunnel_session, + register_conversation_key, + register_tunnel_session, + route_message_to_rust, + unregister_conversation_key, + unregister_tunnel_session, +) + +_WINDOW_SECONDS = ephemeral_gateway_timeout_ms(True) / 1000.0 + + +class TestRouteMessageToRustGatewayError(unittest.TestCase): + def setUp(self): + self.conversation_id = 'conv-under-test' + self.tube_id = 'tube-under-test' + register_conversation_key(self.conversation_id, b'dummy-symmetric-key') + self.session = TunnelSession( + tube_id=self.tube_id, conversation_id=self.conversation_id, + gateway_uid='gw-uid', symmetric_key=b'dummy-symmetric-key', + ) + register_tunnel_session(self.tube_id, self.session) + + def tearDown(self): + unregister_conversation_key(self.conversation_id) + unregister_tunnel_session(self.tube_id) + + def _gateway_error_response(self, error_text): + return { + 'conversationId': self.conversation_id, + 'payload': json.dumps({'is_ok': False, 'data': error_text}), + } + + def _make_registry(self): + tube_registry = mock.Mock() + tube_registry.tube_id_from_connection_id.return_value = self.tube_id + return tube_registry + + def test_auth_failed_within_window_closes_and_unregisters_tube(self): + tube_registry = self._make_registry() + + route_message_to_rust( + self._gateway_error_response('gateway_webrtcaction_keeperssh_proxy_auth_failed'), + tube_registry, + ) + + tube_registry.close_tube.assert_called_once_with( + self.tube_id, reason=CloseConnectionReasons.Normal) + self.assertIsNone(get_tunnel_session(self.tube_id)) + + def test_timeout_within_window_closes_tube(self): + tube_registry = self._make_registry() + + route_message_to_rust( + self._gateway_error_response('gateway_webrtcaction_keeperssh_proxy_timeout'), + tube_registry, + ) + + tube_registry.close_tube.assert_called_once_with( + self.tube_id, reason=CloseConnectionReasons.Normal) + + def test_config_error_within_window_closes_tube(self): + tube_registry = self._make_registry() + + route_message_to_rust( + self._gateway_error_response('gateway_webrtcaction_keeperssh_proxy_config_error'), + tube_registry, + ) + + tube_registry.close_tube.assert_called_once_with( + self.tube_id, reason=CloseConnectionReasons.Normal) + + def test_generic_proxy_error_code_does_not_close_tube(self): + # The generic `..._keeperssh_proxy_error` code (missing module / + # missing credentials / unexpected error) is covered by earlier, + # synchronous validation -- not in scope for this async fallback. + tube_registry = self._make_registry() + + route_message_to_rust( + self._gateway_error_response('gateway_webrtcaction_keeperssh_proxy_error'), + tube_registry, + ) + + tube_registry.close_tube.assert_not_called() + self.assertIsNotNone(get_tunnel_session(self.tube_id)) + + def test_unrelated_gateway_error_does_not_close_tube(self): + # E.g. a 401 on get_leafs for an unrelated action on the same + # conversation -- must not tear down an otherwise-healthy tunnel. + tube_registry = self._make_registry() + + route_message_to_rust(self._gateway_error_response('some unrelated error'), tube_registry) + + tube_registry.close_tube.assert_not_called() + + def test_auth_failed_after_window_does_not_close_tube(self): + # Simulate a long-running, already-established tunnel: creation_time + # is older than the startup grace window, so even a matching fatal + # code must not close it. + self.session.creation_time -= (_WINDOW_SECONDS + 5) + tube_registry = self._make_registry() + + route_message_to_rust( + self._gateway_error_response('gateway_webrtcaction_keeperssh_proxy_auth_failed'), + tube_registry, + ) + + tube_registry.close_tube.assert_not_called() + self.assertIsNotNone(get_tunnel_session(self.tube_id)) + + def test_matching_code_still_stashes_message_when_outside_window(self): + self.session.creation_time -= (_WINDOW_SECONDS + 5) + tube_registry = self._make_registry() + + route_message_to_rust( + self._gateway_error_response('gateway_webrtcaction_keeperssh_proxy_timeout'), + tube_registry, + ) + + self.assertEqual(self.session.gateway_error_message, 'gateway_webrtcaction_keeperssh_proxy_timeout') + self.assertTrue(self.session.gateway_error_event.is_set()) + + def test_gateway_error_still_stashes_message_for_synchronous_waiter(self): + # Even for a non-matching/non-fatal error, the message/event stash + # must still happen so a synchronous waiter still in its retry + # window picks up the real error text instead of a generic timeout. + tube_registry = self._make_registry() + + route_message_to_rust(self._gateway_error_response('specific failure reason'), tube_registry) + + self.assertEqual(self.session.gateway_error_message, 'specific failure reason') + self.assertTrue(self.session.gateway_error_event.is_set()) + + def test_no_tube_found_does_not_raise(self): + tube_registry = mock.Mock() + tube_registry.tube_id_from_connection_id.return_value = None + + route_message_to_rust( + self._gateway_error_response('gateway_webrtcaction_keeperssh_proxy_auth_failed'), + tube_registry, + ) + + tube_registry.close_tube.assert_not_called() + + def test_close_tube_exception_does_not_prevent_unregister(self): + tube_registry = self._make_registry() + tube_registry.close_tube.side_effect = Exception('tube already gone') + + route_message_to_rust( + self._gateway_error_response('gateway_webrtcaction_keeperssh_proxy_auth_failed'), + tube_registry, + ) + + self.assertIsNone(get_tunnel_session(self.tube_id)) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/service/test_cloudflare_config.py b/unit-tests/service/test_cloudflare_config.py new file mode 100644 index 000000000..9701073ec --- /dev/null +++ b/unit-tests/service/test_cloudflare_config.py @@ -0,0 +1,20 @@ +import unittest + +from keepercommander.service.config.cloudflare_config import CloudflareConfigurator +from keepercommander.service.util.tunneling import get_tunnel_log_file + + +class TestCloudflareLogPath(unittest.TestCase): + def test_health_check_reads_the_same_file_the_tunnel_writes_to(self): + """_get_cloudflare_log_path() used to hardcode its own copy of the log path, + which drifted out of sync when the tunnel log location moved to the shared + service_logs dir - the health check kept reading the old, now-empty file and + always timed out with 'status could not be determined'.""" + self.assertEqual( + CloudflareConfigurator._get_cloudflare_log_path(), + get_tunnel_log_file("cloudflare_tunnel_subprocess.log"), + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/service/test_command.py b/unit-tests/service/test_command.py index 2df0637b1..c1ffdbf1b 100644 --- a/unit-tests/service/test_command.py +++ b/unit-tests/service/test_command.py @@ -1,10 +1,40 @@ +import json import unittest from unittest import TestCase, mock from flask import Flask +from keepercommander import params as params_module, vault +from keepercommander.subfolder import RootFolderNode, SharedFolderNode from keepercommander.service.util.command_util import CommandExecutor from keepercommander.service.util.exceptions import CommandExecutionError from keepercommander.service.util.parse_keeper_response import parse_keeper_response +from keepercommander.service.util.protected_records import get_protected_record_uids + +PROTECTED_TITLE = 'Commander Service Mode Config' +PROTECTED_UID = 'PROTECTED_CONFIG_UID' +NORMAL_UID = 'NORMAL_RECORD_UID' + + +def _record_cache_entry(uid, title): + return { + 'record_uid': uid, + 'version': 2, + 'revision': 1, + 'client_modified_time': 0, + 'shared': False, + 'record_key_unencrypted': b'0' * 32, + 'data_unencrypted': json.dumps({'title': title}).encode('utf-8'), + } + + +def _params_with_protected_and_normal_record(): + p = params_module.KeeperParams() + p.service_mode = False + p.record_cache = { + PROTECTED_UID: _record_cache_entry(PROTECTED_UID, PROTECTED_TITLE), + NORMAL_UID: _record_cache_entry(NORMAL_UID, 'My Normal Record'), + } + return p class TestCommandAPI(TestCase): def setUp(self): @@ -115,4 +145,658 @@ def test_integration_command_flow(self): response, status_code = CommandExecutor.execute(test_command) self.assertEqual(status_code, 200) - self.assertIsNotNone(response) \ No newline at end of file + self.assertIsNotNone(response) + + +class TestProtectedRecordCommandExecution(TestCase): + """End-to-end proof, through CommandExecutor.execute, that Service Mode's own config record stays blocked while unrelated records work.""" + + def _run(self, command, params=None): + params = params or _params_with_protected_and_normal_record() + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', '') + ) as mock_capture: + response, status_code = CommandExecutor.execute(command) + return response, status_code, mock_capture, params + + def test_blocked_commands_never_reach_cli_dispatch(self): + for command in ( + f'get {PROTECTED_UID}', + f'get "{PROTECTED_TITLE}"', + f'list {PROTECTED_UID}', + f'search {PROTECTED_UID}', + f'record-update --record {PROTECTED_UID} title=x', + f'record-update --record={PROTECTED_UID} title=x', + f'rm {PROTECTED_UID}', + f'share-record {PROTECTED_UID} --email a@b.com', + f'share-folder --record {PROTECTED_UID} -e a@b.com', + f'share-folder --record={PROTECTED_UID} -e a@b.com', + f'ls "{PROTECTED_TITLE}"', + f'tree "{PROTECTED_TITLE}"', + f'nsf-get {PROTECTED_UID}', + # Not one of the commands anyone would think to curate a list around -- + # proves the check isn't gated by command name at all. + f'keep-alive {PROTECTED_UID}', + ): + with self.subTest(command=command): + response, status_code, mock_capture, _ = self._run(command) + self.assertEqual(status_code, 403) + self.assertEqual(response.get('status'), 'error') + mock_capture.assert_not_called() + + def test_unrelated_record_commands_are_unaffected(self): + for command in ( + f'get {NORMAL_UID}', + f'list {NORMAL_UID}', + f'search {NORMAL_UID}', + f'record-update --record {NORMAL_UID} title=x', + f'rm {NORMAL_UID}', + f'share-record {NORMAL_UID} --email a@b.com', + ): + with self.subTest(command=command): + response, status_code, mock_capture, _ = self._run(command) + self.assertEqual(status_code, 200) + mock_capture.assert_called_once() + + def test_record_cache_is_popped_during_dispatch_and_restored_after(self): + params = _params_with_protected_and_normal_record() + seen_during_call = {} + + def fake_capture(p, command): + seen_during_call['keys'] = set(p.record_cache.keys()) + return 'ok', 'ok', '' + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object(CommandExecutor, 'capture_output_and_logs', side_effect=fake_capture): + response, status_code = CommandExecutor.execute(f'get {NORMAL_UID}') + + self.assertEqual(status_code, 200) + self.assertNotIn(PROTECTED_UID, seen_during_call['keys']) + self.assertIn(NORMAL_UID, seen_during_call['keys']) + # Restored after the call returns -- the shared params singleton must + # not stay permanently blind to its own config record. + self.assertIn(PROTECTED_UID, params.record_cache) + + def test_record_cache_restored_even_if_dispatch_raises(self): + params = _params_with_protected_and_normal_record() + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', side_effect=CommandExecutionError('boom') + ): + response, status_code = CommandExecutor.execute(f'get {NORMAL_UID}') + + self.assertEqual(status_code, 400) + self.assertIn(PROTECTED_UID, params.record_cache) + + def test_protected_record_check_runs_for_every_command(self): + """The check is unconditional -- even a command with no known relationship + to records (whoami) still triggers it, so no command can be missed.""" + params = _params_with_protected_and_normal_record() + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', '') + ), mock.patch( + 'keepercommander.service.util.command_util.get_protected_record_uids', + wraps=get_protected_record_uids, + ) as mock_get_uids: + CommandExecutor.execute('whoami') + mock_get_uids.assert_called_once() + + def test_sailpoint_handling_runs_before_the_record_cache_guard(self): + """handle_command needs its OWN record visible to read its marker/capability fields. + Safe because Layer B already blocks direct references, and _before_share only resolves exact cache keys.""" + params = _params_with_protected_and_normal_record() + seen_during_handle_command = {} + seen_during_dispatch = {} + + def fake_handle_command(p, command): + seen_during_handle_command['keys'] = set(p.record_cache.keys()) + return command, None + + def fake_capture(p, command): + seen_during_dispatch['keys'] = set(p.record_cache.keys()) + return 'ok', 'ok', '' + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.dict('os.environ', {'SAILPOINT_RECORD': PROTECTED_UID}), mock.patch( + 'keepercommander.service.commands.integrations.sailpoint.service.SailPointService.handle_command', + side_effect=fake_handle_command, + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', side_effect=fake_capture, + ): + CommandExecutor.execute(f'get {NORMAL_UID}') + + # Unhidden for SailPoint's own gate -- it needs to read its own config record. + self.assertIn(PROTECTED_UID, seen_during_handle_command['keys']) + self.assertIn(NORMAL_UID, seen_during_handle_command['keys']) + # Hidden again for the actual dispatched command -- an admin still can't reach it. + self.assertNotIn(PROTECTED_UID, seen_during_dispatch['keys']) + self.assertIn(NORMAL_UID, seen_during_dispatch['keys']) + # Restored after the whole request completes. + self.assertIn(PROTECTED_UID, params.record_cache) + + def test_other_protected_records_stay_hidden_from_sailpoint_handle_command(self): + """Regression test: only SailPoint's own pinned UID is exempt from handle_command's guard; + every other protected record (PROTECTED_UID standing in for e.g. Docker's) must stay hidden.""" + params = _params_with_protected_and_normal_record() + seen_during_handle_command = {} + + def fake_handle_command(p, command): + seen_during_handle_command['keys'] = set(p.record_cache.keys()) + return command, None + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.dict('os.environ', {'SAILPOINT_RECORD': 'sailpoint-own-uid'}), mock.patch( + 'keepercommander.service.commands.integrations.sailpoint.service.SailPointService.handle_command', + side_effect=fake_handle_command, + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', ''), + ): + CommandExecutor.execute(f'get {NORMAL_UID}') + + self.assertNotIn(PROTECTED_UID, seen_during_handle_command['keys']) + self.assertIn(NORMAL_UID, seen_during_handle_command['keys']) + + def test_direct_reference_to_protected_record_never_reaches_sailpoint_handle_command(self): + """Layer B blocks a command that directly names the protected UID/title before + SailPoint's handle_command ever runs, regardless of the record being unhidden for it.""" + params = _params_with_protected_and_normal_record() + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.dict('os.environ', {'SAILPOINT_RECORD': 'sailpoint-uid'}), mock.patch( + 'keepercommander.service.commands.integrations.sailpoint.service.SailPointService.handle_command', + ) as mock_handle_command, mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', ''), + ) as mock_capture: + response, status_code = CommandExecutor.execute(f'get {PROTECTED_UID}') + + self.assertEqual(status_code, 403) + mock_handle_command.assert_not_called() + mock_capture.assert_not_called() + + def test_sailpoint_marker_check_failure_denies_the_request(self): + """A transient marker-read failure must deny the request, not silently skip SailPoint's gating.""" + params = _params_with_protected_and_normal_record() + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.dict('os.environ', {'SAILPOINT_RECORD': 'sailpoint-uid'}), mock.patch( + 'keepercommander.service.commands.integrations.sailpoint.service.SailPointService.record_has_marker', + side_effect=RuntimeError('decrypt failed'), + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', ''), + ) as mock_capture: + response, status_code = CommandExecutor.execute(f'get {NORMAL_UID}') + + self.assertEqual(status_code, 500) + mock_capture.assert_not_called() + + def test_sailpoint_own_record_stays_hidden_during_dispatch_when_another_record_is_also_protected(self): + """Regression test: when a second protected record forces the outer (handle_command) + guard to actually wrap record_cache, the inner (dispatch) guard must still hide + SailPoint's own UID too, not silently no-op because the cache is no longer a plain dict.""" + sailpoint_uid = 'SAILPOINT_OWN_UID' + params = params_module.KeeperParams() + params.service_mode = False + params.record_cache = { + sailpoint_uid: _record_cache_entry(sailpoint_uid, 'Commander Service Mode SailPoint Config'), + PROTECTED_UID: _record_cache_entry(PROTECTED_UID, PROTECTED_TITLE), + NORMAL_UID: _record_cache_entry(NORMAL_UID, 'My Normal Record'), + } + seen_during_dispatch = {} + + def fake_capture(p, command): + seen_during_dispatch['keys'] = set(p.record_cache.keys()) + return 'ok', 'ok', '' + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.dict('os.environ', {'SAILPOINT_RECORD': sailpoint_uid}), mock.patch( + 'keepercommander.service.commands.integrations.sailpoint.service.SailPointService.handle_command', + side_effect=lambda p, command: (command, None), + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', side_effect=fake_capture, + ): + CommandExecutor.execute(f'get {NORMAL_UID}') + + self.assertNotIn(sailpoint_uid, seen_during_dispatch['keys']) + self.assertNotIn(PROTECTED_UID, seen_during_dispatch['keys']) + self.assertIn(NORMAL_UID, seen_during_dispatch['keys']) + + def test_other_protected_records_stay_hidden_during_sailpoint_after_command(self): + """after_command follows the same rule as handle_command: SailPoint's own record stays + visible (it writes the pending-entitlement queue there), every other protected record stays hidden.""" + sailpoint_uid = 'SAILPOINT_OWN_UID' + params = params_module.KeeperParams() + params.service_mode = False + params.record_cache = { + sailpoint_uid: _record_cache_entry(sailpoint_uid, 'Commander Service Mode SailPoint Config'), + PROTECTED_UID: _record_cache_entry(PROTECTED_UID, PROTECTED_TITLE), + NORMAL_UID: _record_cache_entry(NORMAL_UID, 'My Normal Record'), + } + seen_during_after_command = {} + + def fake_after_command(p, command, success=True): + seen_during_after_command['keys'] = set(p.record_cache.keys()) + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.dict('os.environ', {'SAILPOINT_RECORD': sailpoint_uid}), mock.patch( + 'keepercommander.service.commands.integrations.sailpoint.service.SailPointService.handle_command', + side_effect=lambda p, command: (command, None), + ), mock.patch( + 'keepercommander.service.commands.integrations.sailpoint.service.SailPointService.after_command', + side_effect=fake_after_command, + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', ''), + ): + CommandExecutor.execute(f'get {NORMAL_UID}') + + self.assertIn(sailpoint_uid, seen_during_after_command['keys']) + self.assertNotIn(PROTECTED_UID, seen_during_after_command['keys']) + self.assertIn(NORMAL_UID, seen_during_after_command['keys']) + + +class TestSyncDownExemptionCommandExecution(TestCase): + """slack-app-setup --sync-down must keep working for its OWN config record while every + other protected record (including a different integration's) stays blocked.""" + + SLACK_UID = 'SLACK_CONFIG_UID' + GCHAT_UID = 'GCHAT_CONFIG_UID' + + def _params(self): + p = params_module.KeeperParams() + p.service_mode = False + p.record_cache = { + self.SLACK_UID: _record_cache_entry(self.SLACK_UID, 'Commander Service Mode Slack App Config'), + self.GCHAT_UID: _record_cache_entry(self.GCHAT_UID, 'Commander Service Mode Google Chat App Config'), + NORMAL_UID: _record_cache_entry(NORMAL_UID, 'My Normal Record'), + } + return p + + def _run(self, command, params, capture_side_effect=None): + env = {'SLACK_RECORD': self.SLACK_UID, 'GCHAT_RECORD': self.GCHAT_UID} + with mock.patch.dict('os.environ', env), mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', + side_effect=capture_side_effect, return_value=('ok', 'ok', '') if capture_side_effect is None else None, + ) as mock_capture: + response, status_code = CommandExecutor.execute(command) + return response, status_code, mock_capture + + def test_get_slack_record_is_blocked(self): + params = self._params() + response, status_code, mock_capture = self._run(f'get {self.SLACK_UID}', params) + self.assertEqual(status_code, 403) + mock_capture.assert_not_called() + + def test_slack_sync_down_default_flow_is_not_blocked_and_sees_its_own_record(self): + params = self._params() + seen = {} + + def fake_capture(p, command): + seen['keys'] = set(p.record_cache.keys()) + return 'ok', 'ok', '' + + response, status_code, mock_capture = self._run( + 'slack-app-setup --sync-down', params, capture_side_effect=fake_capture + ) + self.assertEqual(status_code, 200) + mock_capture.assert_called_once() + self.assertIn(self.SLACK_UID, seen['keys']) # not hidden for this one dispatch + self.assertIn(self.GCHAT_UID, params.record_cache) # untouched throughout + + def test_slack_sync_down_with_explicit_own_uid_is_not_blocked(self): + params = self._params() + response, status_code, mock_capture = self._run( + f'slack-app-setup --sync-down -r {self.SLACK_UID}', params + ) + self.assertEqual(status_code, 200) + mock_capture.assert_called_once() + + def test_slack_sync_down_with_a_different_integrations_uid_is_still_blocked(self): + params = self._params() + response, status_code, mock_capture = self._run( + f'slack-app-setup --sync-down -r {self.GCHAT_UID}', params + ) + self.assertEqual(status_code, 403) + mock_capture.assert_not_called() + + def test_slack_setup_without_sync_down_gets_no_exemption(self): + """The main (non --sync-down) flow must not get the record cache exemption + just because it names the record's own UID.""" + params = self._params() + response, status_code, mock_capture = self._run( + f'slack-app-setup --slack-record-name {self.SLACK_UID}', params + ) + self.assertEqual(status_code, 403) + mock_capture.assert_not_called() + + def test_gchat_sync_down_default_flow_is_not_blocked_and_sees_its_own_record(self): + """Same as Slack's own-record flow, but for GChat -- proves the exemption isn't Slack-specific.""" + params = self._params() + seen = {} + + def fake_capture(p, command): + seen['keys'] = set(p.record_cache.keys()) + return 'ok', 'ok', '' + + response, status_code, mock_capture = self._run( + 'gchat-app-setup --sync-down', params, capture_side_effect=fake_capture + ) + self.assertEqual(status_code, 200) + mock_capture.assert_called_once() + self.assertIn(self.GCHAT_UID, seen['keys']) + self.assertIn(self.SLACK_UID, params.record_cache) + + def test_gchat_sync_down_with_a_different_integrations_uid_is_still_blocked(self): + params = self._params() + response, status_code, mock_capture = self._run( + f'gchat-app-setup --sync-down -r {self.SLACK_UID}', params + ) + self.assertEqual(status_code, 403) + mock_capture.assert_not_called() + + +class TestTerraformRecordProtectionCommandExecution(TestCase): + """Same UID-pinning protection Docker/Slack/GChat get, proven at the CommandExecutor.execute() boundary.""" + + TERRAFORM_UID = 'TERRAFORM_CONFIG_UID' + + def _params(self): + p = params_module.KeeperParams() + p.service_mode = False + p.record_cache = { + self.TERRAFORM_UID: _record_cache_entry(self.TERRAFORM_UID, 'Commander Service Mode Terraform Config'), + NORMAL_UID: _record_cache_entry(NORMAL_UID, 'My Normal Record'), + } + return p + + def _run(self, command, params): + with mock.patch.dict('os.environ', {'TERRAFORM_RECORD': self.TERRAFORM_UID}), mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', '') + ) as mock_capture: + response, status_code = CommandExecutor.execute(command) + return response, status_code, mock_capture + + def test_get_terraform_record_is_blocked(self): + params = self._params() + response, status_code, mock_capture = self._run(f'get {self.TERRAFORM_UID}', params) + self.assertEqual(status_code, 403) + mock_capture.assert_not_called() + + def test_share_record_on_terraform_record_is_blocked(self): + params = self._params() + response, status_code, mock_capture = self._run( + f'share-record {self.TERRAFORM_UID} --email a@b.com', params + ) + self.assertEqual(status_code, 403) + mock_capture.assert_not_called() + + def test_normal_record_is_unaffected(self): + params = self._params() + response, status_code, mock_capture = self._run(f'get {NORMAL_UID}', params) + self.assertEqual(status_code, 200) + mock_capture.assert_called_once() + + +class TestProtectedFolderCommandExecution(TestCase): + """The shared folder holding a protected config record must be just as unreachable + as the record itself -- ls/tree/rndir/mv/share-folder all resolve folders through the + same caches hide_from_folder_cache guards.""" + + PROTECTED_FOLDER_UID = 'PROTECTED_FOLDER_UID' + PROTECTED_FOLDER_TITLE = 'Commander Service Mode - Docker' + NORMAL_FOLDER_UID = 'NORMAL_FOLDER_UID' + + def _params(self): + p = params_module.KeeperParams() + p.service_mode = False + p.record_cache = { + PROTECTED_UID: _record_cache_entry(PROTECTED_UID, PROTECTED_TITLE), + NORMAL_UID: _record_cache_entry(NORMAL_UID, 'My Normal Record'), + } + p.root_folder = RootFolderNode() + + protected_node = SharedFolderNode() + protected_node.uid = self.PROTECTED_FOLDER_UID + protected_node.name = self.PROTECTED_FOLDER_TITLE + normal_node = SharedFolderNode() + normal_node.uid = self.NORMAL_FOLDER_UID + normal_node.name = 'My Normal Folder' + + p.folder_cache = {self.PROTECTED_FOLDER_UID: protected_node, self.NORMAL_FOLDER_UID: normal_node} + p.root_folder.subfolders = [self.PROTECTED_FOLDER_UID, self.NORMAL_FOLDER_UID] + p.shared_folder_cache = { + self.PROTECTED_FOLDER_UID: {'name_unencrypted': self.PROTECTED_FOLDER_TITLE}, + self.NORMAL_FOLDER_UID: {'name_unencrypted': 'My Normal Folder'}, + } + p.subfolder_cache = { + self.PROTECTED_FOLDER_UID: {'type': 'shared_folder', 'shared_folder_uid': self.PROTECTED_FOLDER_UID}, + self.NORMAL_FOLDER_UID: {'type': 'shared_folder', 'shared_folder_uid': self.NORMAL_FOLDER_UID}, + } + p.subfolder_record_cache = { + self.PROTECTED_FOLDER_UID: {PROTECTED_UID}, + self.NORMAL_FOLDER_UID: {NORMAL_UID}, + } + return p + + def _run(self, command, params): + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', '') + ) as mock_capture: + response, status_code = CommandExecutor.execute(command) + return response, status_code, mock_capture + + def test_blocked_folder_commands_never_reach_cli_dispatch(self): + """Layer B's literal-token scan is UID-only for folders (by design -- see the plan); + a title-only reference isn't caught here, it fails to resolve at all once Layer A + hides the folder from folder_cache/shared_folder_cache, proven separately in + TestHideFromFolderCache and test_folder_caches_hidden_during_dispatch_and_restored_after.""" + for command in ( + f'ls {self.PROTECTED_FOLDER_UID}', + f'tree {self.PROTECTED_FOLDER_UID}', + f'rndir {self.PROTECTED_FOLDER_UID} x', + f'mv {self.PROTECTED_FOLDER_UID} /', + f'share-folder {self.PROTECTED_FOLDER_UID} -e a@b.com', + ): + with self.subTest(command=command): + params = self._params() + response, status_code, mock_capture = self._run(command, params) + self.assertEqual(status_code, 403) + mock_capture.assert_not_called() + + def test_normal_folder_commands_are_unaffected(self): + params = self._params() + response, status_code, mock_capture = self._run(f'ls {self.NORMAL_FOLDER_UID}', params) + self.assertEqual(status_code, 200) + mock_capture.assert_called_once() + + def test_folder_caches_hidden_during_dispatch_and_restored_after(self): + params = self._params() + seen = {} + + def fake_capture(p, command): + seen['folder_keys'] = set(p.folder_cache.keys()) + seen['subfolders'] = list(p.root_folder.subfolders) + return 'ok', 'ok', '' + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object(CommandExecutor, 'capture_output_and_logs', side_effect=fake_capture): + response, status_code = CommandExecutor.execute(f'ls {self.NORMAL_FOLDER_UID}') + + self.assertEqual(status_code, 200) + self.assertNotIn(self.PROTECTED_FOLDER_UID, seen['folder_keys']) + self.assertIn(self.NORMAL_FOLDER_UID, seen['folder_keys']) + self.assertNotIn(self.PROTECTED_FOLDER_UID, seen['subfolders']) + + self.assertIn(self.PROTECTED_FOLDER_UID, params.folder_cache) + self.assertIn(self.PROTECTED_FOLDER_UID, params.root_folder.subfolders) + + +class TestSailPointFolderProtectionCommandExecution(TestCase): + """Regression test: hide_from_folder_cache was imported and fed protected_folder_uids, + but never actually invoked as a context manager, so a protected record's folder kept + showing up in tree/ls even though the record itself was correctly hidden.""" + + FOLDER_UID = 'PROTECTED_FOLDER_UID' + RECORD_UID = 'PROTECTED_FOLDER_RECORD_UID' + + def _params(self): + p = params_module.KeeperParams() + p.service_mode = False + p.record_cache = { + self.RECORD_UID: _record_cache_entry(self.RECORD_UID, 'Commander Service Mode SailPoint Config'), + NORMAL_UID: _record_cache_entry(NORMAL_UID, 'My Normal Record'), + } + p.root_folder = RootFolderNode() + node = SharedFolderNode() + node.uid = self.FOLDER_UID + node.name = 'Commander Service Mode - SailPoint' + p.folder_cache = {self.FOLDER_UID: node} + p.root_folder.subfolders = [self.FOLDER_UID] + p.shared_folder_cache = {self.FOLDER_UID: {'name_unencrypted': node.name}} + p.subfolder_cache = {self.FOLDER_UID: {'type': 'shared_folder', 'shared_folder_uid': self.FOLDER_UID}} + p.subfolder_record_cache = {self.FOLDER_UID: {self.RECORD_UID}} + return p + + def test_folder_hidden_during_actual_dispatch(self): + params = self._params() + seen = {} + + def fake_capture(p, command): + seen['folder_keys'] = set(p.folder_cache.keys()) + seen['subfolders'] = list(p.root_folder.subfolders) + return 'ok', 'ok', '' + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.object(CommandExecutor, 'capture_output_and_logs', side_effect=fake_capture): + CommandExecutor.execute(f'get {NORMAL_UID}') + + self.assertNotIn(self.FOLDER_UID, seen['folder_keys']) + self.assertNotIn(self.FOLDER_UID, seen['subfolders']) + self.assertIn(self.FOLDER_UID, params.folder_cache) + self.assertIn(self.FOLDER_UID, params.root_folder.subfolders) + + def test_folder_hidden_during_sailpoint_handle_command_too(self): + """Unlike the record, the folder has no reason to be visible to handle_command, + so it must stay hidden for the whole request, not just the final dispatch.""" + params = self._params() + seen = {} + + def fake_handle_command(p, command): + seen['folder_keys'] = set(p.folder_cache.keys()) + return command, None + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch.dict('os.environ', {'SAILPOINT_RECORD': 'sailpoint-uid'}), mock.patch( + 'keepercommander.service.commands.integrations.sailpoint.service.SailPointService.handle_command', + side_effect=fake_handle_command, + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', '') + ): + CommandExecutor.execute(f'get {NORMAL_UID}') + + self.assertNotIn(self.FOLDER_UID, seen['folder_keys']) + + +class TestReservedAttachmentCommandExecution(TestCase): + """A record with an arbitrary, non-default title must still be blocked if it carries + one of Commander's own reserved config-file attachments (config.json/service_config.json).""" + + ARBITRARY_UID = 'ARBITRARY_TITLED_RECORD_UID' + + def _params(self): + p = params_module.KeeperParams() + p.service_mode = False + p.record_cache = { + self.ARBITRARY_UID: _record_cache_entry(self.ARBITRARY_UID, 'My Totally Unrelated Title'), + NORMAL_UID: _record_cache_entry(NORMAL_UID, 'My Normal Record'), + } + return p + + @staticmethod + def _record_with_reserved_attachment(uid, title): + record = vault.PasswordRecord() + record.record_uid = uid + record.title = title + record.attachments = [vault.AttachmentFile({'id': f'{uid}_ATTA', 'name': 'config.json'})] + return record + + @staticmethod + def _plain_record(uid, title): + record = vault.PasswordRecord() + record.record_uid = uid + record.title = title + return record + + _TITLES = {ARBITRARY_UID: 'My Totally Unrelated Title', NORMAL_UID: 'My Normal Record'} + + def _run(self, command, params, reserved_uids=(ARBITRARY_UID,)): + records = { + uid: ( + self._record_with_reserved_attachment(uid, self._TITLES[uid]) + if uid in reserved_uids else self._plain_record(uid, self._TITLES[uid]) + ) + for uid in params.record_cache + } + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch( + 'keepercommander.vault.KeeperRecord.load', side_effect=lambda p, uid: records.get(uid) + ), mock.patch.object( + CommandExecutor, 'capture_output_and_logs', return_value=('ok', 'ok', '') + ) as mock_capture: + response, status_code = CommandExecutor.execute(command) + return response, status_code, mock_capture + + def test_get_on_record_with_reserved_attachment_is_blocked(self): + params = self._params() + response, status_code, mock_capture = self._run(f'get {self.ARBITRARY_UID}', params) + self.assertEqual(status_code, 403) + mock_capture.assert_not_called() + + def test_file_report_omits_record_with_reserved_attachment(self): + params = self._params() + records = { + self.ARBITRARY_UID: self._record_with_reserved_attachment( + self.ARBITRARY_UID, self._TITLES[self.ARBITRARY_UID] + ), + NORMAL_UID: self._plain_record(NORMAL_UID, self._TITLES[NORMAL_UID]), + } + seen = {} + + def fake_capture(p, command): + seen['keys'] = set(p.record_cache.keys()) + return 'ok', 'ok', '' + + with mock.patch( + 'keepercommander.service.core.globals.ensure_params_loaded', return_value=params + ), mock.patch( + 'keepercommander.vault.KeeperRecord.load', side_effect=lambda p, uid: records.get(uid) + ), mock.patch.object(CommandExecutor, 'capture_output_and_logs', side_effect=fake_capture): + response, status_code = CommandExecutor.execute('file-report') + + self.assertEqual(status_code, 200) + self.assertNotIn(self.ARBITRARY_UID, seen['keys']) + self.assertIn(NORMAL_UID, seen['keys']) + + def test_record_without_reserved_attachment_is_unaffected(self): + params = self._params() + response, status_code, mock_capture = self._run(f'get {NORMAL_UID}', params, reserved_uids=()) + self.assertEqual(status_code, 200) + mock_capture.assert_called_once() diff --git a/unit-tests/service/test_command_policy.py b/unit-tests/service/test_command_policy.py new file mode 100644 index 000000000..a562faeb8 --- /dev/null +++ b/unit-tests/service/test_command_policy.py @@ -0,0 +1,59 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | '