
    z-PhSV                     &   d dl Z d dlmZ d dlZ	 d dlmZ d dlmc m	Z
 d dlmZmZ n# e$ r dZdZ
Y nw xY wdZdZdZdZdZe j        j        e j        j        gZ e j        d	
          d             Z e j        d	
          d             Zd Zd Zd Zd Zd Zd Zd Z d Z!d Z"d Z#d Z$d Z%d Z&d Z'd Z(d Z)d Z*e j        +                    d          d              Z,e j        +                    d!          d"             Z-d# Z.d$ Z/d% Z0dS )&    N)	timedelta)InMemoryKmsClientverify_file_encryptedzencrypted_table.in_mem.parquets   0123456789112345
footer_keys   1234567890123450col_keymodule)scopec                      t           j                            t          j        g d          t          j        g d          t          j        g d          d          } | S )N)         )abc)xyz)paTablefrom_pydictarray)
data_tables    e/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pyarrow/tests/parquet/test_encryption.pyr   r   0   s^    %%Xiii  Xooo&&Xooo&&' '  J
     c                  N    t          j        t          t          ddgi          } | S )Nr   r   )r   column_keys)peEncryptionConfigurationFOOTER_KEY_NAMECOL_KEY_NAME)basic_encryption_configs    r   r!   r!   :   s3     8"3*
  
 #"r   c                 b    t          j        |           }d }t          j        |          }||fS )z
    Sets up and returns the KMS connection configuration and crypto factory
    based on provided KMS configuration parameters.
    custom_kms_confc                      t          |           S Nr   kms_connection_configurations    r   kms_factoryz1setup_encryption_environment.<locals>.kms_factoryK        !=>>>r   )r   KmsConnectionConfigCryptoFactory)r$   kms_connection_configr*   crypto_factorys       r   setup_encryption_environmentr0   D   sD    
 2?SSS? ? ? %k22N .00r   c                     ||                     d          ||                     d          i}t          |          \  }}	t          | ||||	           ||	fS )zL
    Writes an encrypted parquet file based on the provided parameters.
    UTF-8)decoder0   write_encrypted_parquet)
pathr   footer_key_namecol_key_namer   r   encryption_configr$   r.   r/   s
             r   write_encrypted_filer9   T   sz     	**733gnnW--O -I- -)> D*.?1>C C C !.00r   c           	         | t           z  }t          j        t          t          ddgidt          d          d          }t          ||t          t          t          t          |          \  }}t          |           t          j
        t          d                    }t          ||||          }|                    |          sJ d	S )
zDWrite an encrypted parquet, verify it's encrypted, and then read it.r   r   
AES_GCM_V1      @minutes   r   r   encryption_algorithmcache_lifetimedata_key_length_bitsrB   N)PARQUET_NAMEr   r   r   r    r   r9   
FOOTER_KEYCOL_KEYr   DecryptionConfigurationread_encrypted_parquetequals)tempdirr   r5   r8   r.   r/   decryption_configresult_tables           r   !test_encrypted_parquet_write_readrN   j   s    \!D
 2"3*
 * --- " " " -Aj/<W- -)> $ 2 ---/ / /)!6H HL\*******r   c                     |                     ||          }|J t          j        | |j        |          5 }|                    |           d d d            d S # 1 swxY w Y   d S )N)encryption_properties)file_encryption_propertiespqParquetWriterschemawrite_table)r5   tabler8   r.   r/   rQ   writers          r   r4   r4      s    !/!J!J0"2 "2%111		%,"<
> 
> 
> "AG5!!!" " " " " " " " " " " " " " " " " "s   AA!Ac                 4   |                     ||          }|J t          j        | |          }|j        dk    sJ t          j        | |          }t          |j                  dk    sJ t          j        | |          }|                    d          S )Ndecryption_propertiesr   Tuse_threads)	file_decryption_propertiesrR   read_metadatanum_columnsread_schemalennamesParquetFileread)r5   rL   r.   r/   r]   metarT   results           r   rI   rI      s    !/!J!J0"2 "2%111$>@ @ @Dq    ^$>@ @ @Fv|!!!!^$>@ @ @F;;4;(((r   c           	      f   | t           z  }t          j        t          t          ddgidt          d          d          }t          ||t          t          t          t          |           t          |           t          t          t                              d          t          t                              d          i          \  }}t          j        t          d          	          }t          j        t          d
          5  t!          ||||           ddd           dS # 1 swxY w Y   dS )zYWrite an encrypted parquet, verify it's encrypted,
    and then read it using wrong keys.r   r   r;   r<   r=   r?   r@   r2   rD   zIncorrect master key usedmatchN)rE   r   r   r   r    r   r9   rF   rG   r   r0   r3   rH   pytestraises
ValueErrorrI   )rK   r   r5   r8   wrong_kms_connection_configwrong_crypto_factoryrL   s          r   +test_encrypted_parquet_write_read_wrong_keyro      s    \!D
 2"3*
 * --- " " " z?L#W.?A A A $8T00j''00V 9 95!5
 2 ---/ / /	z)E	F	F	F " "#%@ 	" 	" 	"" " " " " " " " " " " " " " " " " "s   D&&D*-D*c                     t          | |           t          j        t          d          5  t	          j        | t          z                                             ddd           dS # 1 swxY w Y   dS )zmWrite an encrypted parquet, verify it's encrypted,
    but then try to read it without decryption properties.no decryptionrh   N)rN   rj   rk   IOErrorrR   rc   rE   rd   rK   r   s     r   0test_encrypted_parquet_read_no_decryption_configrt      s     &gz:::	w&6	7	7	7 6 6
w-..335556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   /A((A,/A,c                     t          | |           t          j        t          d          5  t	          j        | t          z             ddd           dS # 1 swxY w Y   dS )zwWrite an encrypted parquet, verify it's encrypted,
    but then try to read its metadata without decryption properties.rq   rh   N)rN   rj   rk   rr   rR   r^   rE   rs   s     r   9test_encrypted_parquet_read_metadata_no_decryption_configrv      s     &gz:::	w&6	7	7	7 1 1
</0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1   AAAc                     t          | |           t          j        t          d          5  t	          j        | t          z             ddd           dS # 1 swxY w Y   dS )zuWrite an encrypted parquet, verify it's encrypted,
    but then try to read its schema without decryption properties.rq   rh   N)rN   rj   rk   rr   rR   r`   rE   rs   s     r   7test_encrypted_parquet_read_schema_no_decryption_configry      s     &gz:::	w&6	7	7	7 / /
w-.../ / / / / / / / / / / / / / / / / /rw   c           
          | dz  }t          j        t                    }t          j        t
          d          5  t          ||t          t          t          d|           ddd           dS # 1 swxY w Y   dS )zMWrite an encrypted parquet, but give only footer key,
    without column key.z)encrypted_table_no_col_key.in_mem.parquetr   z4Either column_keys or uniform_encryption must be setrh   r   N)	r   r   r   rj   rk   OSErrorr9   r    rF   )rK   r   r5   r8   s       r   'test_encrypted_parquet_write_no_col_keyr}      s     @@D 2"$ $ $ 
w%
& 
& 
& A A 	T:'.?	A 	A 	A	A A A A A A A A A A A A A A A A A As   %A--A14A1c                     | dz  }|}t          j                    }d }t          j        |          }t          j        t
          d          5  t          |||||           ddd           dS # 1 swxY w Y   dS )<Write an encrypted parquet, but raise KeyError in KmsClient.(encrypted_table_kms_error.in_mem.parquetc                      t          |           S r&   r'   r(   s    r   r*   z;test_encrypted_parquet_write_kms_error.<locals>.kms_factory   s     !!=>>>r   r   rh   N)r   r,   r-   rj   rk   KeyErrorr4   rK   r   r!   r5   r8   r.   r*   r/   s           r   &test_encrypted_parquet_write_kms_errorr      s     ??D/ 244? ? ?
 %k22N	x|	4	4	4 G Gj2C 5~	G 	G 	GG G G G G G G G G G G G G G G G G G   A..A25A2c                 ,   | dz  }|}t          j                    } G d dt           j                  fd}t          j        |          }t	          j        t          d          5  t          |||||           ddd           dS # 1 swxY w Y   dS )r   r   c                   $    e Zd ZdZd Zd Zd ZdS )Jtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClientzVA KmsClient implementation that throws exception in
        wrap/unwrap calls
        c                 R    t           j                            |            || _        dS )z%Create an InMemoryKmsClient instance.N)r   	KmsClient__init__configselfr   s     r   r   zStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.__init__  s#    L!!$''' DKKKr   c                      t          d          )NCannot Wrap Keyrl   r   	key_bytesmaster_key_identifiers      r   wrap_keyzStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.wrap_key  s    .///r   c                      t          d          )NzCannot Unwrap Keyr   r   wrapped_keyr   s      r   
unwrap_keyzUtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.unwrap_key  s    0111r   N__name__
__module____qualname____doc__r   r   r    r   r   ThrowingKmsClientr     sK        	 		! 	! 	!
	0 	0 	0	2 	2 	2 	2 	2r   r   c                      |           S r&   r   )r)   r   s    r   r*   zDtest_encrypted_parquet_write_kms_specific_error.<locals>.kms_factory"  s      !=>>>r   r   rh   N)r   r,   r   r-   rj   rk   rl   r4   )	rK   r   r!   r5   r8   r.   r*   r/   r   s	           @r   /test_encrypted_parquet_write_kms_specific_errorr   	  s-    ??D/ 2442 2 2 2 2BL 2 2 2 ? ? ? ? ? %k22N	z):	;	;	; G Gj2C 5~	G 	G 	GG G G G G G G G G G G G G G G G G Gs   (B		BBc                     | dz  }|}t          j                    }d }t          j        |          }t          j        t
          d          5  t          |||||           ddd           dS # 1 swxY w Y   dS )z@Write an encrypted parquet, but raise ValueError in kms_factory.0encrypted_table_kms_factory_error.in_mem.parquetc                      t          d          )NCannot create KmsClientr   r(   s    r   r*   zCtest_encrypted_parquet_write_kms_factory_error.<locals>.kms_factory6  s    2333r   r   rh   N)r   r,   r-   rj   rk   rl   r4   r   s           r   .test_encrypted_parquet_write_kms_factory_errorr   -  s     GGD/ 2444 4 4 %k22N	z6
8 
8 
8 G G 	 j2C 5~	G 	G 	GG G G G G G G G G G G G G G G G G Gr   c                    | dz  }|}t          j                    } G d d          fd}t          j        |          }t          j        t
                    5  t          |||||           ddd           dS # 1 swxY w Y   dS )z_Write an encrypted parquet, but use wrong KMS client type
    that doesn't implement KmsClient.r   c                   $    e Zd ZdZd Zd Zd ZdS )Otest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClientz4This is not an implementation of KmsClient.
        c                     |j         | _        d S r&   )r$   master_keys_mapr   s     r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.__init__O  s    #)#9D   r   c                     d S r&   r   r   s      r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.wrap_keyR      4r   c                     d S r&   r   r   s      r   r   zZtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.unwrap_keyU  r   r   Nr   r   r   r   WrongTypeKmsClientr   K  sK        	 		: 	: 	:	 	 		 	 	 	 	r   r   c                      |           S r&   r   )r)   r   s    r   r*   zHtest_encrypted_parquet_write_kms_factory_type_error.<locals>.kms_factoryX  s    !!">???r   N)r   r,   r-   rj   rk   	TypeErrorr4   )	rK   r   r!   r5   r8   r.   r*   r/   r   s	           @r   3test_encrypted_parquet_write_kms_factory_type_errorr   A  s(    GGD/ 244       @ @ @ @ @ %k22N	y	!	! G Gj2C 5~	G 	G 	GG G G G G G G G G G G G G G G G G Gs   A<<B B c            
      l   d } t          j        t          t          ddgidddt	          d          dd	
          } | |           t          j        t                    }t          ddgi|_        d|_        d|_        d|_        t	          d          |_	        d|_
        d	|_         | |           d S )Nc                     t           | j        k    sJ ddg| j        t                   k    sJ d| j        k    sJ | j        sJ | j        rJ t          d          | j        k    sJ | j	        rJ d| j
        k    sJ d S )Nr   r   AES_GCM_CTR_V1      $@r=      )r   r   r   r    rA   plaintext_footerdouble_wrappingr   rB   internal_key_materialrC   )r8   s    r   !validate_encryption_configurationzZtest_encrypted_parquet_encryption_configuration.<locals>.validate_encryption_configurationc  s    "3">>>>>Sz.:<HHHHH#4#IIIII 1111$4444&&&*;*JJJJJ$::::'<<<<<<<r   r   r   r   TFr   r=   r   )r   r   rA   r   r   rB   r   rC   r{   )r   r   r   r    r   r   rA   r   r   rB   r   rC   )r   r8   encryption_config_1s      r   /test_encrypted_parquet_encryption_configurationr   b  s    = = = 2"!C:0- ...# 	 	 	 &%&78884"$ $ $'3c3Z&B#/?,+/(*/')24)@)@)@&05-/2,%%&9:::::r   c                     t          j        t          d                    } t          d          | j        k    sJ t          j                    }t          d          |_        t          d          |j        k    sJ d S )Nr   r=   rD   )r   rH   r   rB   )rL   decryption_config_1s     r   /test_encrypted_parquet_decryption_configurationr     s    2 ...0 0 0T"""&7&FFFFF466)24)@)@)@&T"""&9&HHHHHHHr   c                      d } t          j        dddddd          } | |           t          j                    }d|_        d|_        d|_        ddd|_         | |           d S )	Nc                 t    d| j         k    sJ d| j        k    sJ d| j        k    sJ ddd| j        k    sJ d S )N	Instance1URL1MyTokenkey_material_1key_material_2key1key2kms_instance_idkms_instance_urlkey_access_tokenr$   )r.   s    r   validate_kms_connection_configzPtest_encrypted_parquet_kms_configuration.<locals>.validate_kms_connection_config  sq    3CCCCC.?????1BBBBB)3CDD%56 6 6 6 6 6r   r   r   r   r   r   r   r   )r   r,   r   r   r   r$   )r   r.   kms_connection_config_1s      r   (test_encrypted_parquet_kms_configurationr     s    7 7 7 2#"$$
 
	   #"#8999 466.9+/5,/8,  / /+ #"#:;;;;;r   zNPlaintext footer - reading plaintext column subset reads encrypted columns too)reasonc                 `   | t           z  }t          j        t          t          ddgidd          }t          j        t          t                              d          t          t                              d          i          }d }t          j	        |          }t          |||||           d	S )
zWrite an encrypted parquet, with plaintext footer
    and with single wrapping,
    verify it's encrypted, and then read plaintext columns.r   r   TF)r   r   r   r   r2   r#   c                      t          |           S r&   r'   r(   s    r   r*   zStest_encrypted_parquet_write_read_plain_footer_single_wrapping.<locals>.kms_factory  r+   r   N)rE   r   r   r   r    r,   rF   r3   rG   r-   r4   rK   r   r5   r8   r.   r*   r/   s          r   >test_encrypted_parquet_write_read_plain_footer_single_wrappingr     s     \!D
 2"3*
    2Z..w77'..11
  ? ? ? %k22ND*.?1>C C C C Cr   z'External key material not supported yetc                    | t           z  }t          j        t          i d          }t          j        t          t
                              d          i          }d }t          j        |          }t          |||||           dS )zWrite an encrypted parquet, with external key
    material.
    Currently it's not implemented, so should throw
    an exceptionF)r   r   r   r2   r#   c                      t          |           S r&   r'   r(   s    r   r*   z:test_encrypted_parquet_write_external.<locals>.kms_factory  r+   r   N)	rE   r   r   r   r,   rF   r3   r-   r4   r   s          r   %test_encrypted_parquet_write_externalr     s     \!D 2"#% % %
 2(**;*;G*D*DE  ? ? ? %k22ND*.?1>C C C C Cr   c           	         | t           z  }t          ||t          t          t          t
          |          \  }}t          |           t          j        t          d                    }t          d          D ]_}|                    ||          }|J t          j        ||          }	|	                    d          }
|                    |
          sJ `dS )	z`Write an encrypted parquet, verify it's encrypted,
    and then read it multithreaded in a loop.r<   r=   rD   2   NrY   Tr[   )rE   r9   r   r    rF   rG   r   r   rH   r   ranger]   rR   rc   rd   rJ   )rK   r   r!   r5   r.   r/   rL   ir]   rf   rM   s              r   test_encrypted_parquet_loopr     s    \!D
 -Aj/<W-! -!)> $2 ---/ / / 2YY 	/ 	/%3%N%N!#4&6 &6")555(BD D D{{t{44  ......	/ 	/r   c           	         | t           z  }t          ||t          t          t          t
          |          \  }}t          |           t          j        t          d                    }|
                    ||          }~t          j        ||          }|                    d          }	|                    |	          sJ dS )z^
    Test that decryption properties can be used if the crypto factory is no longer alive
    r<   r=   rD   rY   Tr[   N)rE   r9   r   r    rF   rG   r   r   rH   r   r]   rR   rc   rd   rJ   )
rK   r   r!   r5   r.   r/   rL   r]   rf   rM   s
             r   %test_read_with_deleted_crypto_factoryr     s     \!D,@j/<W-! -!)> $ 2 ---/ / /!/!J!J0"2 "2^$>@ @ @F;;4;00L\*******r   c           	         | t           z  }t          ||t          t          t          t
          |          \  }}t          j        t          d                    }|	                    ||          }t          j        ||          }|                    |          sJ t          j        | |          }|                    |          sJ dS )z>Write an encrypted parquet then read it back using read_table.r<   r=   rD   rY   N)rE   r9   r   r    rF   rG   r   rH   r   r]   rR   
read_tablerJ   )	rK   r   r!   r5   r.   r/   rL   r]   rM   s	            r   !test_encrypted_parquet_read_tabler   %  s    \!D -Aj/<W-! -!)> 2 ---/ / /!/!J!J0"2 "2 ==WXXXL \***** ='AC C CL\*******r   )1rj   datetimer   pyarrowr   pyarrow.parquetparquetrR   pyarrow.parquet.encryption
encryptionr    pyarrow.tests.parquet.encryptionr   r   ImportErrorrE   rF   r   rG   r    markparquet_encryption
pytestmarkfixturer   r!   r0   r9   rN   r4   rI   ro   rt   rv   ry   r}   r   r   r   r   r   r   r   xfailr   r   r   r   r   r   r   r   <module>r      s  "           2      +++++++++
2 2 2 2 2 2 2 2 2	    	B	BBB 0 

 K"
K
 h    h# #  #1 1 1 1 1 1,+ + +<" " ") ) )" "  "  "F6 6 61 1 1/ / /A A A"G G G*!G !G !GHG G G(G G GB ;  ;  ;FI I I< < <:  2 3 3C C3 3CN CDDC C EDC4/ / /:+ + +0+ + + + +s   ( 	44