관리-도구
편집 파일: spark_driver.cpython-38.pyc
U �Mg� � @ s� d dl Z d dlmZ d dlmZmZ d dlmZ er\d dlmZ d dlm Z d dl m Z m Z G dd � d e�Z d d � Zd d � Zdd� ZG dd� d�ZG dd� de�ZdS )� N)� Integration)�capture_internal_exceptions�ensure_integration_enabled)� TYPE_CHECKING)�Any)�Optional)�Event�Hintc @ s e Zd ZdZedd� �ZdS )�SparkIntegrationZsparkc C s t � d S �N)�patch_spark_context_init� r r �X/opt/hc_python/lib/python3.8/site-packages/sentry_sdk/integrations/spark/spark_driver.py� setup_once s zSparkIntegration.setup_onceN)�__name__� __module__� __qualname__� identifier� staticmethodr r r r r r s r c C s6 ddl m} | j}|r2|�d|j� |�d|j� dS )z� Set properties in driver that propagate to worker processes, allowing for workers to have access to those properties. This allows worker integration to have access to app_name and application_id. r �� SparkContextZsentry_app_nameZsentry_application_idN)�pysparkr �_active_spark_contextZsetLocalProperty�appName� applicationId)r Z spark_contextr r r �_set_app_properties s �r c C s4 ddl m} | j}||� t� }| j�� �|� dS )zA Start java gateway server to add custom `SparkListener` r )� ensure_callback_server_startedN)Zpyspark.java_gatewayr Z_gateway�SentryListenerZ_jsc�scZaddSparkListener)r r ZgwZlistenerr r r �_start_sentry_listener' s r c s2 ddl m} | j� tt� �� fdd��}|| _d S )Nr r c s< �� f|�|�}t � � t� t�� }|j� fdd��}|S )Nc s@ t � ��. t�� �t�d kr,| W 5 Q R � S � jd krF| W 5 Q R � S | �di ��d� �� � | �di ��d� j� d�� | d �d� j� d�� | d �d� j� d �� | d �d � j� d �� | d �d � j � | d �d � j � | d �d� j � | d �d� j � | d �d� j� | �di ��d� j� W 5 Q R X | S )N�user�id�tagsz executor.idzspark.executor.idzspark-submit.deployModezspark.submit.deployModez driver.hostzspark.driver.hostz driver.portzspark.driver.portZ spark_versionZapp_nameZapplication_id�masterZ spark_home�extraZweb_url)r � sentry_sdkZ get_clientZget_integrationr r � setdefaultZ sparkUserZ_conf�get�versionr r r# Z sparkHomeZuiWebUrl)�event�hint��selfr r � process_eventC s: � � � � z[patch_spark_context_init.<locals>._sentry_patched_spark_context_init.<locals>.process_event)r r r% Zget_isolation_scopeZadd_event_processor)r, �args�kwargs�rv�scoper- �Zspark_context_initr+ r �"_sentry_patched_spark_context_init: s "zDpatch_spark_context_init.<locals>._sentry_patched_spark_context_init)r r Z_do_initr r )r r3 r r2 r r 4 s -r c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� ZG d1d2� d2�Zd3S )4� SparkListenerc C s d S r r )r, ZapplicationEndr r r �onApplicationEndl s z SparkListener.onApplicationEndc C s d S r r )r, ZapplicationStartr r r �onApplicationStartp s z SparkListener.onApplicationStartc C s d S r r )r, ZblockManagerAddedr r r �onBlockManagerAddedt s z!SparkListener.onBlockManagerAddedc C s d S r r )r, ZblockManagerRemovedr r r �onBlockManagerRemovedx s z#SparkListener.onBlockManagerRemovedc C s d S r r )r, Z blockUpdatedr r r �onBlockUpdated| s z SparkListener.onBlockUpdatedc C s d S r r )r, ZenvironmentUpdater r r �onEnvironmentUpdate� s z!SparkListener.onEnvironmentUpdatec C s d S r r )r, Z executorAddedr r r �onExecutorAdded� s z SparkListener.onExecutorAddedc C s d S r r )r, ZexecutorBlacklistedr r r �onExecutorBlacklisted� s z#SparkListener.onExecutorBlacklistedc C s d S r r )r, ZexecutorBlacklistedForStager r r � onExecutorBlacklistedForStage� s z+SparkListener.onExecutorBlacklistedForStagec C s d S r r )r, ZexecutorMetricsUpdater r r �onExecutorMetricsUpdate� s z%SparkListener.onExecutorMetricsUpdatec C s d S r r )r, ZexecutorRemovedr r r �onExecutorRemoved� s z SparkListener.onExecutorRemovedc C s d S r r )r, �jobEndr r r �onJobEnd� s zSparkListener.onJobEndc C s d S r r )r, �jobStartr r r � onJobStart� s zSparkListener.onJobStartc C s d S r r )r, ZnodeBlacklistedr r r �onNodeBlacklisted� s z SparkListener.onNodeBlacklistedc C s d S r r )r, ZnodeBlacklistedForStager r r �onNodeBlacklistedForStage� s z'SparkListener.onNodeBlacklistedForStagec C s d S r r )r, ZnodeUnblacklistedr r r �onNodeUnblacklisted� s z!SparkListener.onNodeUnblacklistedc C s d S r r )r, r) r r r � onOtherEvent� s zSparkListener.onOtherEventc C s d S r r )r, ZspeculativeTaskr r r �onSpeculativeTaskSubmitted� s z(SparkListener.onSpeculativeTaskSubmittedc C s d S r r )r, �stageCompletedr r r �onStageCompleted� s z SparkListener.onStageCompletedc C s d S r r )r, �stageSubmittedr r r �onStageSubmitted� s z SparkListener.onStageSubmittedc C s d S r r )r, ZtaskEndr r r � onTaskEnd� s zSparkListener.onTaskEndc C s d S r r )r, ZtaskGettingResultr r r �onTaskGettingResult� s z!SparkListener.onTaskGettingResultc C s d S r r )r, Z taskStartr r r � onTaskStart� s zSparkListener.onTaskStartc C s d S r r )r, Z unpersistRDDr r r �onUnpersistRDD� s z SparkListener.onUnpersistRDDc @ s e Zd ZdgZdS )zSparkListener.Javaz1org.apache.spark.scheduler.SparkListenerInterfaceN)r r r Z implementsr r r r �Java� s rQ N) r r r r5 r6 r7 r8 r9 r: r; r< r= r> r? rA rC rD rE rF rG rH rJ rL rM rN rO rP rQ r r r r r4 k s2 r4 c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) r c C s& d� |�� �}tjd|d� t� d S )NzJob {} Started�info)�level�message)�format�jobIdr% �add_breadcrumbr )r, rB rT r r r rC � s zSentryListener.onJobStartc C sb d}d}d|� � �� i}|� � �� dkr<d}d�|�� �}nd}d�|�� �}tj|||d� d S ) N� �resultZ JobSucceededrR z Job {} Ended�warningz Job {} Failed�rS rT �data)Z jobResultZtoStringrU rV r% rW )r, r@ rS rT r\ r r r rA � s zSentryListener.onJobEndc C sB |� � }d�|�� �}|�� |�� d�}tjd||d� t� d S )NzStage {} Submitted�� attemptId�namerR r[ )� stageInforU �stageIdr^ r_ r% rW r )r, rK � stage_inforT r\ r r r rL � s z SentryListener.onStageSubmittedc C s� ddl m} |�� }d}d}|�� |�� d�}z&|�� �� |d< d�|�� �}d}W n& |k rz d�|�� �}d }Y nX t j |||d � d S ) Nr )� Py4JJavaErrorrX r] �reasonzStage {} FailedrZ zStage {} CompletedrR r[ ) Z py4j.protocolrc r` r^ r_ Z failureReasonr' rU ra r% rW )r, rI rc rb rT rS r\ r r r rJ � s z SentryListener.onStageCompletedN)r r r rC rA rL rJ r r r r r � s r )r% Zsentry_sdk.integrationsr Zsentry_sdk.utilsr r �typingr r r Zsentry_sdk._typesr r r r r r r4 r r r r r �<module> s 7g