codecraft
codecraft

聊聊flink JobManager的High Availability

序

本文主要研究一下flink JobManager的High Availability

配置

flink-conf.yaml

high-availability: zookeeper
high-availability.zookeeper.quorum: zookeeper:2181
high-availability.zookeeper.path.root: /flink
high-availability.cluster-id: /cluster_one # important: customize per cluster
high-availability.storageDir: file:///share
  • high-availability的可选值为NONE或者zookeeper；high-availability.zookeeper.quorum用于指定zookeeper的peers；high-availability.zookeeper.path.root用于指定在zookeeper的root node路径；high-availability.cluster-id用于指定当前cluster的node名称，该cluster node位于root node下面；high-availability.storageDir用于指定JobManager metadata的存储路径

masters文件

localhost:8081
localhost:8082
  • masters文件用于指定jobmanager的地址

HighAvailabilityMode

flink-runtime_2.11-1.7.1-sources.jar!/org/apache/flink/runtime/jobmanager/HighAvailabilityMode.java

public enum HighAvailabilityMode {
    NONE(false),
    ZOOKEEPER(true),
    FACTORY_CLASS(true);

    private final boolean haActive;

    HighAvailabilityMode(boolean haActive) {
        this.haActive = haActive;
    }

    /**
     * Return the configured {@link HighAvailabilityMode}.
     *
     * @param config The config to parse
     * @return Configured recovery mode or {@link HighAvailabilityMode#NONE} if not
     * configured.
     */
    public static HighAvailabilityMode fromConfig(Configuration config) {
        String haMode = config.getValue(HighAvailabilityOptions.HA_MODE);

        if (haMode == null) {
            return HighAvailabilityMode.NONE;
        } else if (haMode.equalsIgnoreCase(ConfigConstants.DEFAULT_RECOVERY_MODE)) {
            // Map old default to new default
            return HighAvailabilityMode.NONE;
        } else {
            try {
                return HighAvailabilityMode.valueOf(haMode.toUpperCase());
            } catch (IllegalArgumentException e) {
                return FACTORY_CLASS;
            }
        }
    }

    /**
     * Returns true if the defined recovery mode supports high availability.
     *
     * @param configuration Configuration which contains the recovery mode
     * @return true if high availability is supported by the recovery mode, otherwise false
     */
    public static boolean isHighAvailabilityModeActivated(Configuration configuration) {
        HighAvailabilityMode mode = fromConfig(configuration);
        return mode.haActive;
    }
}
  • HighAvailabilityMode有三个枚举，分别是NONE、ZOOKEEPER、FACTORY_CLASS；这些枚举有一个属性haActive，用于表示是否支持HighAvailability

HighAvailabilityOptions

flink-core-1.7.1-sources.jar!/org/apache/flink/configuration/HighAvailabilityOptions.java

@PublicEvolving
@ConfigGroups(groups = {
    @ConfigGroup(name = "HighAvailabilityZookeeper", keyPrefix = "high-availability.zookeeper")
})
public class HighAvailabilityOptions {

    // ------------------------------------------------------------------------
    //  Required High Availability Options
    // ------------------------------------------------------------------------

    /**
     * Defines high-availability mode used for the cluster execution.
     * A value of "NONE" signals no highly available setup.
     * To enable high-availability, set this mode to "ZOOKEEPER".
     * Can also be set to FQN of HighAvailability factory class.
     */
    @Documentation.CommonOption(position = Documentation.CommonOption.POSITION_HIGH_AVAILABILITY)
    public static final ConfigOption HA_MODE =
            key("high-availability")
            .defaultValue("NONE")
            .withDeprecatedKeys("recovery.mode")
            .withDescription("Defines high-availability mode used for the cluster execution." +
                " To enable high-availability, set this mode to \"ZOOKEEPER\" or specify FQN of factory class.");

    /**
     * The ID of the Flink cluster, used to separate multiple Flink clusters
     * Needs to be set for standalone clusters, is automatically inferred in YARN and Mesos.
     */
    public static final ConfigOption HA_CLUSTER_ID =
            key("high-availability.cluster-id")
            .defaultValue("/default")
            .withDeprecatedKeys("high-availability.zookeeper.path.namespace", "recovery.zookeeper.path.namespace")
            .withDescription("The ID of the Flink cluster, used to separate multiple Flink clusters from each other." +
                " Needs to be set for standalone clusters but is automatically inferred in YARN and Mesos.");

    /**
     * File system path (URI) where Flink persists metadata in high-availability setups.
     */
    @Documentation.CommonOption(position = Documentation.CommonOption.POSITION_HIGH_AVAILABILITY)
    public static final ConfigOption HA_STORAGE_PATH =
            key("high-availability.storageDir")
            .noDefaultValue()
            .withDeprecatedKeys("high-availability.zookeeper.storageDir", "recovery.zookeeper.storageDir")
            .withDescription("File system path (URI) where Flink persists metadata in high-availability setups.");

    // ------------------------------------------------------------------------
    //  Recovery Options
    // ------------------------------------------------------------------------

    /**
     * Optional port (range) used by the job manager in high-availability mode.
     */
    public static final ConfigOption HA_JOB_MANAGER_PORT_RANGE =
            key("high-availability.jobmanager.port")
            .defaultValue("0")
            .withDeprecatedKeys("recovery.jobmanager.port")
            .withDescription("Optional port (range) used by the job manager in high-availability mode.");

    /**
     * The time before a JobManager after a fail over recovers the current jobs.
     */
    public static final ConfigOption HA_JOB_DELAY =
            key("high-availability.job.delay")
            .noDefaultValue()
            .withDeprecatedKeys("recovery.job.delay")
            .withDescription("The time before a JobManager after a fail over recovers the current jobs.");

    // ------------------------------------------------------------------------
    //  ZooKeeper Options
    // ------------------------------------------------------------------------

    /**
     * The ZooKeeper quorum to use, when running Flink in a high-availability mode with ZooKeeper.
     */
    public static final ConfigOption HA_ZOOKEEPER_QUORUM =
            key("high-availability.zookeeper.quorum")
            .noDefaultValue()
            .withDeprecatedKeys("recovery.zookeeper.quorum")
            .withDescription("The ZooKeeper quorum to use, when running Flink in a high-availability mode with ZooKeeper.");

    /**
     * The root path under which Flink stores its entries in ZooKeeper.
     */
    public static final ConfigOption HA_ZOOKEEPER_ROOT =
            key("high-availability.zookeeper.path.root")
            .defaultValue("/flink")
            .withDeprecatedKeys("recovery.zookeeper.path.root")
            .withDescription("The root path under which Flink stores its entries in ZooKeeper.");

    public static final ConfigOption HA_ZOOKEEPER_LATCH_PATH =
            key("high-availability.zookeeper.path.latch")
            .defaultValue("/leaderlatch")
            .withDeprecatedKeys("recovery.zookeeper.path.latch")
            .withDescription("Defines the znode of the leader latch which is used to elect the leader.");

    /** ZooKeeper root path (ZNode) for job graphs. */
    public static final ConfigOption HA_ZOOKEEPER_JOBGRAPHS_PATH =
            key("high-availability.zookeeper.path.jobgraphs")
            .defaultValue("/jobgraphs")
            .withDeprecatedKeys("recovery.zookeeper.path.jobgraphs")
            .withDescription("ZooKeeper root path (ZNode) for job graphs");

    public static final ConfigOption HA_ZOOKEEPER_LEADER_PATH =
            key("high-availability.zookeeper.path.leader")
            .defaultValue("/leader")
            .withDeprecatedKeys("recovery.zookeeper.path.leader")
            .withDescription("Defines the znode of the leader which contains the URL to the leader and the current" +
                " leader session ID.");

    /** ZooKeeper root path (ZNode) for completed checkpoints. */
    public static final ConfigOption HA_ZOOKEEPER_CHECKPOINTS_PATH =
            key("high-availability.zookeeper.path.checkpoints")
            .defaultValue("/checkpoints")
            .withDeprecatedKeys("recovery.zookeeper.path.checkpoints")
            .withDescription("ZooKeeper root path (ZNode) for completed checkpoints.");

    /** ZooKeeper root path (ZNode) for checkpoint counters. */
    public static final ConfigOption HA_ZOOKEEPER_CHECKPOINT_COUNTER_PATH =
            key("high-availability.zookeeper.path.checkpoint-counter")
            .defaultValue("/checkpoint-counter")
            .withDeprecatedKeys("recovery.zookeeper.path.checkpoint-counter")
            .withDescription("ZooKeeper root path (ZNode) for checkpoint counters.");

    /** ZooKeeper root path (ZNode) for Mesos workers. */
    @PublicEvolving
    public static final ConfigOption HA_ZOOKEEPER_MESOS_WORKERS_PATH =
            key("high-availability.zookeeper.path.mesos-workers")
            .defaultValue("/mesos-workers")
            .withDeprecatedKeys("recovery.zookeeper.path.mesos-workers")
            .withDescription(Description.builder()
                .text("The ZooKeeper root path for persisting the Mesos worker information.")
                .build());

    // ------------------------------------------------------------------------
    //  ZooKeeper Client Settings
    // ------------------------------------------------------------------------

    public static final ConfigOption ZOOKEEPER_SESSION_TIMEOUT =
            key("high-availability.zookeeper.client.session-timeout")
            .defaultValue(60000)
            .withDeprecatedKeys("recovery.zookeeper.client.session-timeout")
            .withDescription("Defines the session timeout for the ZooKeeper session in ms.");

    public static final ConfigOption ZOOKEEPER_CONNECTION_TIMEOUT =
            key("high-availability.zookeeper.client.connection-timeout")
            .defaultValue(15000)
            .withDeprecatedKeys("recovery.zookeeper.client.connection-timeout")
            .withDescription("Defines the connection timeout for ZooKeeper in ms.");

    public static final ConfigOption ZOOKEEPER_RETRY_WAIT =
            key("high-availability.zookeeper.client.retry-wait")
            .defaultValue(5000)
            .withDeprecatedKeys("recovery.zookeeper.client.retry-wait")
            .withDescription("Defines the pause between consecutive retries in ms.");

    public static final ConfigOption ZOOKEEPER_MAX_RETRY_ATTEMPTS =
            key("high-availability.zookeeper.client.max-retry-attempts")
            .defaultValue(3)
            .withDeprecatedKeys("recovery.zookeeper.client.max-retry-attempts")
            .withDescription("Defines the number of connection retries before the client gives up.");

    public static final ConfigOption ZOOKEEPER_RUNNING_JOB_REGISTRY_PATH =
            key("high-availability.zookeeper.path.running-registry")
            .defaultValue("/running_job_registry/");

    public static final ConfigOption ZOOKEEPER_CLIENT_ACL =
            key("high-availability.zookeeper.client.acl")
            .defaultValue("open")
            .withDescription("Defines the ACL (open|creator) to be configured on ZK node. The configuration value can be" +
                " set to “creator” if the ZooKeeper server configuration has the “authProvider” property mapped to use" +
                " SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos).");

    // ------------------------------------------------------------------------

    /** Not intended to be instantiated. */
    private HighAvailabilityOptions() {}
}
  • HighAvailabilityOptions定义了前缀为high-availability.zookeeper的配置项

HighAvailabilityServicesUtils

flink-runtime_2.11-1.7.1-sources.jar!/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java

public class HighAvailabilityServicesUtils {

    public static HighAvailabilityServices createAvailableOrEmbeddedServices(
        Configuration config,
        Executor executor) throws Exception {
        HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config);

        switch (highAvailabilityMode) {
            case NONE:
                return new EmbeddedHaServices(executor);

            case ZOOKEEPER:
                BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

                return new ZooKeeperHaServices(
                    ZooKeeperUtils.startCuratorFramework(config),
                    executor,
                    config,
                    blobStoreService);

            case FACTORY_CLASS:
                return createCustomHAServices(config, executor);

            default:
                throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
        }
    }

    public static HighAvailabilityServices createHighAvailabilityServices(
        Configuration configuration,
        Executor executor,
        AddressResolution addressResolution) throws Exception {

        HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);

        switch (highAvailabilityMode) {
            case NONE:
                final Tuple2 hostnamePort = getJobManagerAddress(configuration);

                final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
                    hostnamePort.f0,
                    hostnamePort.f1,
                    JobMaster.JOB_MANAGER_NAME,
                    addressResolution,
                    configuration);
                final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
                    hostnamePort.f0,
                    hostnamePort.f1,
                    ResourceManager.RESOURCE_MANAGER_NAME,
                    addressResolution,
                    configuration);
                final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
                    hostnamePort.f0,
                    hostnamePort.f1,
                    Dispatcher.DISPATCHER_NAME,
                    addressResolution,
                    configuration);

                final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
                    "%s must be set",
                    RestOptions.ADDRESS.key());
                final int port = configuration.getInteger(RestOptions.PORT);
                final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
                final String protocol = enableSSL ? "https://" : "http://";

                return new StandaloneHaServices(
                    resourceManagerRpcUrl,
                    dispatcherRpcUrl,
                    jobManagerRpcUrl,
                    String.format("%s%s:%s", protocol, address, port));
            case ZOOKEEPER:
                BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

                return new ZooKeeperHaServices(
                    ZooKeeperUtils.startCuratorFramework(configuration),
                    executor,
                    configuration,
                    blobStoreService);

            case FACTORY_CLASS:
                return createCustomHAServices(configuration, executor);

            default:
                throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
        }
    }

    /**
     * Returns the JobManager's hostname and port extracted from the given
     * {@link Configuration}.
     *
     * @param configuration Configuration to extract the JobManager's address from
     * @return The JobManager's hostname and port
     * @throws ConfigurationException if the JobManager's address cannot be extracted from the configuration
     */
    public static Tuple2 getJobManagerAddress(Configuration configuration) throws ConfigurationException {

        final String hostname = configuration.getString(JobManagerOptions.ADDRESS);
        final int port = configuration.getInteger(JobManagerOptions.PORT);

        if (hostname == null) {
            throw new ConfigurationException("Config parameter '" + JobManagerOptions.ADDRESS +
                "' is missing (hostname/address of JobManager to connect to).");
        }

        if (port <= 0 || port >= 65536) {
            throw new ConfigurationException("Invalid value for '" + JobManagerOptions.PORT +
                "' (port of the JobManager actor system) : " + port +
                ".  it must be greater than 0 and less than 65536.");
        }

        return Tuple2.of(hostname, port);
    }

    private static HighAvailabilityServices createCustomHAServices(Configuration config, Executor executor) throws FlinkException {
        final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
        final String haServicesClassName = config.getString(HighAvailabilityOptions.HA_MODE);

        final HighAvailabilityServicesFactory highAvailabilityServicesFactory;

        try {
            highAvailabilityServicesFactory = InstantiationUtil.instantiate(
                haServicesClassName,
                HighAvailabilityServicesFactory.class,
                classLoader);
        } catch (Exception e) {
            throw new FlinkException(
                String.format(
                    "Could not instantiate the HighAvailabilityServicesFactory '%s'. Please make sure that this class is on your class path.",
                    haServicesClassName),
                e);
        }

        try {
            return highAvailabilityServicesFactory.createHAServices(config, executor);
        } catch (Exception e) {
            throw new FlinkException(
                String.format(
                    "Could not create the ha services from the instantiated HighAvailabilityServicesFactory %s.",
                    haServicesClassName),
                e);
        }
    }

    /**
     * Enum specifying whether address resolution should be tried or not when creating the
     * {@link HighAvailabilityServices}.
     */
    public enum AddressResolution {
        TRY_ADDRESS_RESOLUTION,
        NO_ADDRESS_RESOLUTION
    }
}
  • HighAvailabilityServicesUtils提供了创建HighAvailabilityServices的静态方法，这些方法有createAvailableOrEmbeddedServices、createHighAvailabilityServices、createCustomHAServices
  • 其中createAvailableOrEmbeddedServices方法主要是给FlinkMiniCluster使用；createHighAvailabilityServices方法主要是给ClusterEntrypoint使用，它在highAvailabilityMode为NONE的时候创建的是StandaloneHaServices，在highAvailabilityMode为ZOOKEEPER创建的是ZooKeeperHaServices，在highAvailabilityMode为FACTORY_CLASS的时候使用createCustomHAServices方法来创建
  • HighAvailabilityServicesUtils还提供了getJobManagerAddress静态方法，用于获取JobManager的hostname及port

HighAvailabilityServices

flink-runtime_2.11-1.7.1-sources.jar!/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java

/**
 * The HighAvailabilityServices give access to all services needed for a highly-available
 * setup. In particular, the services provide access to highly available storage and
 * registries, as well as distributed counters and leader election.
 * 
 * 
    
 *     
  • ResourceManager leader election and leader retrieval
    • 
 *     
  • JobManager leader election and leader retrieval
    • 
 *     
  • Persistence for checkpoint metadata
    • 
 *     
  • Registering the latest completed checkpoint(s)
    • 
 *     
  • Persistence for the BLOB store
    • 
 *     
  • Registry that marks a job's status
    • 
 *     
  • Naming of RPC endpoints
    • 
 * 

 */
public interface HighAvailabilityServices extends AutoCloseable {

    // ------------------------------------------------------------------------
    //  Constants
    // ------------------------------------------------------------------------

    /**
     * This UUID should be used when no proper leader election happens, but a simple
     * pre-configured leader is used. That is for example the case in non-highly-available
     * standalone setups.
     */
    UUID DEFAULT_LEADER_ID = new UUID(0, 0);

    /**
     * This JobID should be used to identify the old JobManager when using the
     * {@link HighAvailabilityServices}. With the new mode every JobMaster will have a
     * distinct JobID assigned.
     */
    JobID DEFAULT_JOB_ID = new JobID(0L, 0L);

    // ------------------------------------------------------------------------
    //  Services
    // ------------------------------------------------------------------------

    /**
     * Gets the leader retriever for the cluster's resource manager.
     */
    LeaderRetrievalService getResourceManagerLeaderRetriever();

    /**
     * Gets the leader retriever for the dispatcher. This leader retrieval service
     * is not always accessible.
     */
    LeaderRetrievalService getDispatcherLeaderRetriever();

    /**
     * Gets the leader retriever for the job JobMaster which is responsible for the given job
     *
     * @param jobID The identifier of the job.
     * @return Leader retrieval service to retrieve the job manager for the given job
     * @deprecated This method should only be used by the legacy code where the JobManager acts as the master.
     */
    @Deprecated
    LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID);

    /**
     * Gets the leader retriever for the job JobMaster which is responsible for the given job
     *
     * @param jobID The identifier of the job.
     * @param defaultJobManagerAddress JobManager address which will be returned by
     *                              a static leader retrieval service.
     * @return Leader retrieval service to retrieve the job manager for the given job
     */
    LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress);

    LeaderRetrievalService getWebMonitorLeaderRetriever();

    /**
     * Gets the leader election service for the cluster's resource manager.
     *
     * @return Leader election service for the resource manager leader election
     */
    LeaderElectionService getResourceManagerLeaderElectionService();

    /**
     * Gets the leader election service for the cluster's dispatcher.
     *
     * @return Leader election service for the dispatcher leader election
     */
    LeaderElectionService getDispatcherLeaderElectionService();

    /**
     * Gets the leader election service for the given job.
     *
     * @param jobID The identifier of the job running the election.
     * @return Leader election service for the job manager leader election
     */
    LeaderElectionService getJobManagerLeaderElectionService(JobID jobID);

    LeaderElectionService getWebMonitorLeaderElectionService();

    /**
     * Gets the checkpoint recovery factory for the job manager
     *
     * @return Checkpoint recovery factory
     */
    CheckpointRecoveryFactory getCheckpointRecoveryFactory();

    /**
     * Gets the submitted job graph store for the job manager
     *
     * @return Submitted job graph store
     * @throws Exception if the submitted job graph store could not be created
     */
    SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception;

    /**
     * Gets the registry that holds information about whether jobs are currently running.
     *
     * @return Running job registry to retrieve running jobs
     */
    RunningJobsRegistry getRunningJobsRegistry() throws Exception;

    /**
     * Creates the BLOB store in which BLOBs are stored in a highly-available fashion.
     *
     * @return Blob store
     * @throws IOException if the blob store could not be created
     */
    BlobStore createBlobStore() throws IOException;

    // ------------------------------------------------------------------------
    //  Shutdown and Cleanup
    // ------------------------------------------------------------------------

    /**
     * Closes the high availability services, releasing all resources.
     * 
     * 
This method does not delete or clean up any data stored in external stores
     * (file systems, ZooKeeper, etc). Another instance of the high availability
     * services will be able to recover the job.
     * 
     * 
If an exception occurs during closing services, this method will attempt to
     * continue closing other services and report exceptions only after all services
     * have been attempted to be closed.
     *
     * @throws Exception Thrown, if an exception occurred while closing these services.
     */
    @Override
    void close() throws Exception;

    /**
     * Closes the high availability services (releasing all resources) and deletes
     * all data stored by these services in external stores.
     * 
     * 
After this method was called, the any job or session that was managed by
     * these high availability services will be unrecoverable.
     * 
     * 
If an exception occurs during cleanup, this method will attempt to
     * continue the cleanup and report exceptions only after all cleanup steps have
     * been attempted.
     * 
     * @throws Exception Thrown, if an exception occurred while closing these services
     *                   or cleaning up data stored by them.
     */
    void closeAndCleanupAllData() throws Exception;
}
  • HighAvailabilityServices定义了highly-available所需的各种services的get方法

ZooKeeperHaServices

flink-runtime_2.11-1.7.1-sources.jar!/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperHaServices.java

/**
 * An implementation of the {@link HighAvailabilityServices} using Apache ZooKeeper.
 * The services store data in ZooKeeper's nodes as illustrated by teh following tree structure:
 * 
 * 
 * /flink
 *      +/cluster_id_1/resource_manager_lock
 *      |            |
 *      |            +/job-id-1/job_manager_lock
 *      |            |         /checkpoints/latest
 *      |            |                     /latest-1
 *      |            |                     /latest-2
 *      |            |
 *      |            +/job-id-2/job_manager_lock
 *      |      
 *      +/cluster_id_2/resource_manager_lock
 *                   |
 *                   +/job-id-1/job_manager_lock
 *                            |/checkpoints/latest
 *                            |            /latest-1
 *                            |/persisted_job_graph
 * 

 * 
 * 
The root path "/flink" is configurable via the option {@link HighAvailabilityOptions#HA_ZOOKEEPER_ROOT}.
 * This makes sure Flink stores its data under specific subtrees in ZooKeeper, for example to
 * accommodate specific permission.
 * 
 * 
The "cluster_id" part identifies the data stored for a specific Flink "cluster". 
 * This "cluster" can be either a standalone or containerized Flink cluster, or it can be job
 * on a framework like YARN or Mesos (in a "per-job-cluster" mode).
 * 
 * 
In case of a "per-job-cluster" on YARN or Mesos, the cluster-id is generated and configured
 * automatically by the client or dispatcher that submits the Job to YARN or Mesos.
 * 
 * 
In the case of a standalone cluster, that cluster-id needs to be configured via
 * {@link HighAvailabilityOptions#HA_CLUSTER_ID}. All nodes with the same cluster id will join the same
 * cluster and participate in the execution of the same set of jobs.
 */
public class ZooKeeperHaServices implements HighAvailabilityServices {

    private static final Logger LOG = LoggerFactory.getLogger(ZooKeeperHaServices.class);

    private static final String RESOURCE_MANAGER_LEADER_PATH = "/resource_manager_lock";

    private static final String DISPATCHER_LEADER_PATH = "/dispatcher_lock";

    private static final String JOB_MANAGER_LEADER_PATH = "/job_manager_lock";

    private static final String REST_SERVER_LEADER_PATH = "/rest_server_lock";

    // ------------------------------------------------------------------------
    
    
    /** The ZooKeeper client to use */
    private final CuratorFramework client;

    /** The executor to run ZooKeeper callbacks on */
    private final Executor executor;

    /** The runtime configuration */
    private final Configuration configuration;

    /** The zookeeper based running jobs registry */
    private final RunningJobsRegistry runningJobsRegistry;

    /** Store for arbitrary blobs */
    private final BlobStoreService blobStoreService;

    public ZooKeeperHaServices(
            CuratorFramework client,
            Executor executor,
            Configuration configuration,
            BlobStoreService blobStoreService) {
        this.client = checkNotNull(client);
        this.executor = checkNotNull(executor);
        this.configuration = checkNotNull(configuration);
        this.runningJobsRegistry = new ZooKeeperRunningJobsRegistry(client, configuration);

        this.blobStoreService = checkNotNull(blobStoreService);
    }

    // ------------------------------------------------------------------------
    //  Services
    // ------------------------------------------------------------------------

    @Override
    public LeaderRetrievalService getResourceManagerLeaderRetriever() {
        return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
    }

    @Override
    public LeaderRetrievalService getDispatcherLeaderRetriever() {
        return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DISPATCHER_LEADER_PATH);
    }

    @Override
    public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
        return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
    }

    @Override
    public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
        return getJobManagerLeaderRetriever(jobID);
    }

    @Override
    public LeaderRetrievalService getWebMonitorLeaderRetriever() {
        return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, REST_SERVER_LEADER_PATH);
    }

    @Override
    public LeaderElectionService getResourceManagerLeaderElectionService() {
        return ZooKeeperUtils.createLeaderElectionService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
    }

    @Override
    public LeaderElectionService getDispatcherLeaderElectionService() {
        return ZooKeeperUtils.createLeaderElectionService(client, configuration, DISPATCHER_LEADER_PATH);
    }

    @Override
    public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
        return ZooKeeperUtils.createLeaderElectionService(client, configuration, getPathForJobManager(jobID));
    }

    @Override
    public LeaderElectionService getWebMonitorLeaderElectionService() {
        return ZooKeeperUtils.createLeaderElectionService(client, configuration, REST_SERVER_LEADER_PATH);
    }

    @Override
    public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
        return new ZooKeeperCheckpointRecoveryFactory(client, configuration, executor);
    }

    @Override
    public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
        return ZooKeeperUtils.createSubmittedJobGraphs(client, configuration);
    }

    @Override
    public RunningJobsRegistry getRunningJobsRegistry() {
        return runningJobsRegistry;
    }

    @Override
    public BlobStore createBlobStore() throws IOException {
        return blobStoreService;
    }

    // ------------------------------------------------------------------------
    //  Shutdown
    // ------------------------------------------------------------------------

    @Override
    public void close() throws Exception {
        Throwable exception = null;

        try {
            blobStoreService.close();
        } catch (Throwable t) {
            exception = t;
        }

        internalClose();

        if (exception != null) {
            ExceptionUtils.rethrowException(exception, "Could not properly close the ZooKeeperHaServices.");
        }
    }

    @Override
    public void closeAndCleanupAllData() throws Exception {
        LOG.info("Close and clean up all data for ZooKeeperHaServices.");

        Throwable exception = null;

        try {
            blobStoreService.closeAndCleanupAllData();
        } catch (Throwable t) {
            exception = t;
        }

        internalClose();

        if (exception != null) {
            ExceptionUtils.rethrowException(exception, "Could not properly close and clean up all data of ZooKeeperHaServices.");
        }
    }

    /**
     * Closes components which don't distinguish between close and closeAndCleanupAllData
     */
    private void internalClose() {
        client.close();
    }

    // ------------------------------------------------------------------------
    //  Utilities
    // ------------------------------------------------------------------------

    private static String getPathForJobManager(final JobID jobID) {
        return "/" + jobID + JOB_MANAGER_LEADER_PATH;
    }
}
  • ZooKeeperHaServices实现了HighAvailabilityServices接口，它通过ZooKeeperUtils的各种create方法来创建所需的service，比如ZooKeeperUtils.createLeaderRetrievalService、ZooKeeperUtils.createLeaderElectionService、ZooKeeperUtils.createSubmittedJobGraphs

JobClient.submitJob

flink-runtime_2.11-1.7.1-sources.jar!/org/apache/flink/runtime/client/JobClient.java

public class JobClient {

    private static final Logger LOG = LoggerFactory.getLogger(JobClient.class);

    //......

    /**
     * Submits a job to a Flink cluster (non-blocking) and returns a JobListeningContext which can be
     * passed to {@code awaitJobResult} to get the result of the submission.
     * @return JobListeningContext which may be used to retrieve the JobExecutionResult via
     *             {@code awaitJobResult(JobListeningContext context)}.
     */
    public static JobListeningContext submitJob(
            ActorSystem actorSystem,
            Configuration config,
            HighAvailabilityServices highAvailabilityServices,
            JobGraph jobGraph,
            FiniteDuration timeout,
            boolean sysoutLogUpdates,
            ClassLoader classLoader) {

        checkNotNull(actorSystem, "The actorSystem must not be null.");
        checkNotNull(highAvailabilityServices, "The high availability services must not be null.");
        checkNotNull(jobGraph, "The jobGraph must not be null.");
        checkNotNull(timeout, "The timeout must not be null.");

        // for this job, we create a proxy JobClientActor that deals with all communication with
        // the JobManager. It forwards the job submission, checks the success/failure responses, logs
        // update messages, watches for disconnect between client and JobManager, ...

        Props jobClientActorProps = JobSubmissionClientActor.createActorProps(
            highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
            timeout,
            sysoutLogUpdates,
            config);

        ActorRef jobClientActor = actorSystem.actorOf(jobClientActorProps);

        Future submissionFuture = Patterns.ask(
                jobClientActor,
                new JobClientMessages.SubmitJobAndWait(jobGraph),
                new Timeout(AkkaUtils.INF_TIMEOUT()));

        return new JobListeningContext(
            jobGraph.getJobID(),
            submissionFuture,
            jobClientActor,
            timeout,
            classLoader,
            highAvailabilityServices);
    }

    //......
} 
 
     
  
  • 像JobClient.submitJob方法就使用到了highAvailabilityServices.getJobManagerLeaderRetriever方法来获取JobManagerLeader的地址，用于提交job
    •  
 
 
 
小结
 
 
     
  
  • HighAvailabilityMode有三个枚举，分别是NONE、ZOOKEEPER、FACTORY_CLASS；这些枚举有一个属性haActive，用于表示是否支持HighAvailability；HighAvailabilityOptions定义了前缀为high-availability.zookeeper的配置项
    •  
  
  • HighAvailabilityServicesUtils提供了创建HighAvailabilityServices的静态方法，这些方法有createAvailableOrEmbeddedServices、createHighAvailabilityServices、createCustomHAServices；其中createAvailableOrEmbeddedServices方法主要是给FlinkMiniCluster使用；createHighAvailabilityServices方法主要是给ClusterEntrypoint使用，它在highAvailabilityMode为NONE的时候创建的是StandaloneHaServices，在highAvailabilityMode为ZOOKEEPER创建的是ZooKeeperHaServices，在highAvailabilityMode为FACTORY_CLASS的时候使用createCustomHAServices方法来创建
    •  
  
  • HighAvailabilityServices定义了highly-available所需的各种services的get方法；ZooKeeperHaServices实现了HighAvailabilityServices接口，它通过ZooKeeperUtils的各种create方法来创建所需的service，比如ZooKeeperUtils.createLeaderRetrievalService、ZooKeeperUtils.createLeaderElectionService、ZooKeeperUtils.createSubmittedJobGraphs；像JobClient.submitJob方法就使用到了highAvailabilityServices.getJobManagerLeaderRetriever方法来获取JobManagerLeader的地址，用于提交job
    •  
 
 
 
doc
 
 
     
  
  • JobManager High Availability (HA)
    •  
 
 

                            
                        
                    
                    
                    

                    
                    
                    

                
                

                    
                

            
        
    
    

        
你可能感兴趣的:(flink)

        

            

                
    
                    
  • nosql数据库技术与应用知识点
                        皆过客，揽星河
NoSQLnosql数据库大数据数据分析数据结构非关系型数据库
                        
    Nosql知识回顾大数据处理流程数据采集(flume、爬虫、传感器)数据存储(本门课程NoSQL所处的阶段)Hdfs、MongoDB、HBase等数据清洗(入仓)Hive等数据处理、分析(Spark、Flink等)数据可视化数据挖掘、机器学习应用(Python、SparkMLlib等)大数据时代存储的挑战(三高)高并发(同一时间很多人访问)高扩展(要求随时根据需求扩展存储)高效率(要求读写速度快)
    
                    
    • 
                    
  • 全面指南：用户行为从前端数据采集到实时处理的最佳实践
                        数字沉思
营销流量运营系统架构前端内容运营大数据
                        
    引言在当今的数据驱动世界，实时数据采集和处理已经成为企业做出及时决策的重要手段。本文将详细介绍如何通过前端JavaScript代码采集用户行为数据、利用API和Kafka进行数据传输、通过Flink实时处理数据的完整流程。无论你是想提升产品体验还是做用户行为分析，这篇文章都将为你提供全面的解决方案。设计一个通用的ClickHouse表来存储用户事件时，需要考虑多种因素，包括事件类型、时间戳、用户信
    
                    
    • 
                    
  • 详解 Flink 的常见部署方式
                        文刀小桂
Flinkflink大数据
                        
    一、常见部署模式分类1.按是否依赖外部资源调度1.1Standalone模式独立模式(Standalone)是独立运行的，不依赖任何外部的资源管理平台，只需要运行所有Flink组件服务1.2Yarn模式Yarn模式是指客户端把Flink应用提交给Yarn的ResourceManager,Yarn的ResourceManager会在Yarn的NodeManager上创建容器。在这些容器上，Flink
    
                    
    • 
                    
  • 大数据之flink与hive
                        星辰_mya
大数据flinkhive
                        
    其实吧我不太想写flink，因为线上经验确实不多，这也是我需要补的地方，没有条件创造条件，先来一篇吧flink：高性能低延迟流批一体的分布式计算框架基于事件时间对实时数据精准处理快速响应支持批处理，高效离线分析和数据挖掘数据仓库的引擎丰富数据源/接收器，集成多种数据存储格式和源，比较常见就是咱们今天的主题hive了checkpoint恢复机制，故障恢复快速恢复计算任务分布式弹性扩展，据业务灵活增加
    
                    
    • 
                    
  • Java中的大数据处理框架对比分析
                        省赚客app开发者
java开发语言
                        
    Java中的大数据处理框架对比分析大家好，我是微赚淘客系统3.0的小编，是个冬天不穿秋裤，天冷也要风度的程序猿！今天，我们将深入探讨Java中常用的大数据处理框架，并对它们进行对比分析。大数据处理框架是现代数据驱动应用的核心，它们帮助企业处理和分析海量数据，以提取有价值的信息。本文将重点介绍ApacheHadoop、ApacheSpark、ApacheFlink和ApacheStorm这四种流行的
    
                    
    • 
                    
  • 一文搞懂 Flink Task 数据交互之数据写源码
                        mn_kw
flink交互java
                        
    一文搞懂FlinkTask数据交互之数据写源码1.RecordWriterOutput2.RecordWriter3.数据分区器ChannelSelector4.数据输出模型ResultPartition5.子模型ResultSubpartition6.本地buffer池LocalBufferPool7.获取buffer8.将buffer添加到ResultSubpartitionFlink重要源码
    
                    
    • 
                    
  • 概率图模型（PGM）综述
                        医学影像处理
概率图模型概率图模型综述
                        
    RefLink:http://www.sigvc.org/bbs/thread-728-1-1.htmlGraphicalModel的基本类型基本的GraphicalModel可以大致分为两个类别：贝叶斯网络(BayesianNetwork)和马尔可夫随机场(MarkovRandomField)。它们的主要区别在于采用不同类型的图来表达变量之间的关系：贝叶斯网络采用有向无环图(DirectedAc
    
                    
    • 
                    
  • Python基础知识进阶之正则表达式_头歌python正则表达式进阶
                        前端陈萨龙
程序员python学习面试
                        
    最后硬核资料：关注即可领取PPT模板、简历模板、行业经典书籍PDF。技术互助：技术群大佬指点迷津，你的问题可能不是问题，求资源在群里喊一声。面试题库：由技术群里的小伙伴们共同投稿，热乎的大厂面试真题，持续更新中。知识体系：含编程语言、算法、大数据生态圈组件（Mysql、Hive、Spark、Flink）、数据仓库、Python、前端等等。网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是
    
                    
    • 
                    
  • 大数据新视界 --大数据大厂之Flink强势崛起：大数据新视界的璀璨明珠
                        青云交
大数据新视界Flink大数据数据类型实时处理流处理框架对比应用场景数据处理大数据新视界数据库
                        
    亲爱的朋友们，热烈欢迎你们来到青云交的博客！能与你们在此邂逅，我满心欢喜，深感无比荣幸。在这个瞬息万变的时代，我们每个人都在苦苦追寻一处能让心灵安然栖息的港湾。而我的博客，正是这样一个温暖美好的所在。在这里，你们不仅能够收获既富有趣味又极为实用的内容知识，还可以毫无拘束地畅所欲言，尽情分享自己独特的见解。我真诚地期待着你们的到来，愿我们能在这片小小的天地里共同成长，共同进步。本博客的精华专栏：Ja
    
                    
    • 
                    
  • flink增量检查点降低状态依赖实现的详细步骤
                        goTsHgo
Flink大数据分布式flink大数据
                        
    增量检查点启动恢复的时间是很久的，业务上不能接受，所以可以通过降低状态依赖来减少恢复的时间。降低状态依赖尽可能减少状态的复杂性和依赖关系，通过拆分状态或将状态外部化到其他服务中，从而降低恢复的开销。实施措施：将状态分割为更小的单元，减少每次恢复的状态量。使用外部状态存储服务，减少Flink状态后端的负担。拆分状态和将状态外部化到其他服务可以帮助减少作业的状态依赖，从而降低恢复时间和复杂度。以下是详
    
                    
    • 
                    
  • flink table factory基础知识
                        loukey_j

                        
    一、概述在flink中很多组件都是TableFactory的子类。比如序列化，反序列化，tableSinkFactory,tableSourceFactory.TableFactory是用来创建序列化，反序列器，tableSource和tableSink的工厂。二、TableFactory源码在flink框架中，TableFactory的子类并不是程序员自己随心new出来的。flink的提供给程序
    
                    
    • 
                    
  • 2024年最全使用Python求解方程_python解方程(1)，字节面试官迟到
                        2401_84569545
程序员python学习面试
                        
    最后硬核资料：关注即可领取PPT模板、简历模板、行业经典书籍PDF。技术互助：技术群大佬指点迷津，你的问题可能不是问题，求资源在群里喊一声。面试题库：由技术群里的小伙伴们共同投稿，热乎的大厂面试真题，持续更新中。知识体系：含编程语言、算法、大数据生态圈组件（Mysql、Hive、Spark、Flink）、数据仓库、Python、前端等等。网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是
    
                    
    • 
                    
  • 01-Flink安装部署及入门案例（仅供学习），音视频时代你还不会NDK开发
                        小猪佩琪962
2024年程序员学习flink学习大数据
                        
    先自我介绍一下，小编浙江大学毕业，去过华为、字节跳动等大厂，目前阿里P7深知大多数程序员，想要提升技能，往往是自己摸索成长，但自己不成体系的自学效果低效又漫长，而且极易碰到天花板技术停滞不前！因此收集整理了一份《2024年最新大数据全套学习资料》，初衷也很简单，就是希望能够帮助到想自学提升又不知道该从何学起的朋友。既有适合小白学习的零基础资料，也有适合3年以上经验的小伙伴深入学习提升的进阶课程，涵
    
                    
    • 
                    
  • 比较Spark与Flink
                        傲雪凌霜，松柏长青
大数据后端sparkflink大数据
                        
    ApacheSpark和ApacheFlink都是目前非常流行的大数据处理引擎，但它们在架构、处理模式、应用场景等方面有一些显著的区别。下面是二者的对比：1.处理模式Spark:主要支持批处理（BatchProcessing），也能通过SparkStreaming处理流式数据，但SparkStreaming本质上是通过微批（micro-batching）的方式处理流数据，延迟相对较高。SparkS
    
                    
    • 
                    
  • Apache Flink：实时流处理与批处理的统一框架
                        小码快撩
flink大数据
                        
    导语在大数据处理领域，流处理和批处理是两种主要的处理方式。然而，传统的系统通常将这两者视为独立的任务，需要不同的工具和框架来处理。ApacheFlink是一个开源的流处理框架，它打破了这种界限，提供了一个统一的平台来处理实时流数据和批处理数据。一、基本概念与架构ApacheFlink的基本概念与架构主要包括以下几个核心组成部分：基本概念1.流处理模型：无界流(UnboundedStreams):数
    
                    
    • 
                    
  • flink独立集群部署
                        嘎子吱吱吱吱
flinkhadooplinux
                        
    #flink独立集群部署说明安装环境三台服务器47.106.23.1（master）47.112.173.2（worker1）47.115.162.3（worker1）提前装好jdk和ssh,以下操作最好不要用root账号提前下载好flink的包并解压设置三台服务器之间ssh免密登录生成本机秘钥以47.106.23.1为例（其他两台参考本服务器）#生成本机秘钥cd;ssh-keygen-trsa-
    
                    
    • 
                    
  • Flink的时间与watermarks详解
                        大数据技术与数仓

                        
    当我们在使用Flink的时候，避免不了要和时间(time)、水位线(watermarks)打交道，理解这些概念是开发分布式流处理应用的基础。那么Flink支持哪些时间语义？Flink是如何处理乱序事件的？什么是水位线？水位线是如何生成的？水位线的传播方式是什么？让我们带着这些问题来开始本文的内容。时间语义基本概念时间是Flink等流处理中最重要的概念之一，在Flink中Time可以分为三种：Eve
    
                    
    • 
                    
  • 实时数仓之实时数仓架构(Hudi)(1)
                        2401_84164527
程序员架构
                        
    目前比较流行的实时数仓架构有两类，其中一类是以Flink+Doris为核心的实时数仓架构方案；另一类是以湖仓一体架构为核心的实时数仓架构方案。本文针对Flink+Hudi湖仓一体架构进行介绍，这套架构的特点是可以基于一套数据完全实现Lambda架构。实时数仓架构图如下：技术框架Kafka：用于接入数据源；FlinkCDC：如果直接接入业务数据源可以考虑CDC方式，如果通过Kafka缓冲接入业务数据
    
                    
    • 
                    
  • 2024年大数据最新实时数仓之实时数仓架构(Hudi)
                        2401_84185556
程序员大数据架构
                        
    技术框架Kafka：用于接入数据源；FlinkCDC：如果直接接入业务数据源可以考虑CDC方式，如果通过Kafka缓冲接入业务数据可以忽略;Flink：用于数据ETL，包括接入数据、处理数据及输出数据全链路数据计算任务；Spark：用于数据ETL，包括处理数据及输出数据全链路数据计算任务；Hudi：湖仓一体数据管理框架，用来管理模型数据，包括ODS/DWD/DWS/DIM/ADS等；Doris：O
    
                    
    • 
                    
  • 实时数仓之实时数仓架构(Hudi)(1)，2024年最新熬夜整理华为最新大数据开发笔试题
                        2401_84181221
程序员架构大数据
                        
    +Hudi：湖仓一体数据管理框架，用来管理模型数据，包括ODS/DWD/DWS/DIM/ADS等；+Doris：OLAP引擎，同步数仓结果模型，对外提供数据服务支持；+Hbase：用来存储维表信息，维表数据来源一部分有Flink加工实时写入，另一部分是从Spark任务生产，其主要作用用来支持FlinkETL处理过程中的LookupJoin功能。这里选用Hbase原因主要因为Table的HbaseC
    
                    
    • 
                    
  • Flink - CEP
                        kikiki1

                        
    Hadoop3.2集群新版本的搭建详细讲解过程，从下面第一张官方的图来看，最新版是3.2，所以大猪将使用3.2的版本来演示，过程中遇到的坑留给自己，把路留给你们，IT之路还有大猪。大猪为了把文章压缩极简方便小伙伴阅读，将使用root帐号进行所有操作。准备两台主机10.211.55.11、10.211.55.12对应的hostname为m1.example.com、m2.example.com具体命
    
                    
    • 
                    
  • chapter01 Java语言概述 知识点Note
                        月下绯烟
Javajava开发语言
                        
    JavaSEJavaEEJavaME大数据Java基础常用技术栈mysqlJDBCSSMspring+springmvc+mybatisLinuxnacosHadoopFlinkJAVAEE消息队列rabbitMQdocker数据库redisspringbootspringcloudsshstruts+spring+hibernate过时技术栈很少用JAVA虚拟机jvm分布式微服务高并发常见dos
    
                    
    • 
                    
  • 【无标题】大数据之批处理，流处理，批流一体概念
                        数字天下
大数据
                        
    批处理批处理是将一定量的数据集合在一起，形成一个数据批次，然后对这个批次中的数据进行处理。Spark和Flink都支持批处理，其中Spark使用的是批处理模型，即将一批数据一次性读入内存，然后对其进行处理，处理完成后再将结果写入磁盘。Flink也支持批处理，但使用的是基于流处理的批处理模式，即将一批数据分成多个数据流进行处理，可以实现更高效的内存管理和更低的延迟。流处理流式处理是一种将数据流式地处
    
                    
    • 
                    
  • python flink_《Flink官方文档》Python 编程指南测试版
                        weixin_39846361
pythonflink
                        
    原文链接译者：hjjxd校对：清英Flink中的分析程序实现了对数据集的某些操作(例如，数据过滤，映射，合并，分组)。这些数据最初来源于特定的数据源(例如来自于读文件或数据集合)。操作执行的结果通过数据池以写入数据到(分布式)文件系统或标准输出(例如命令行终端)的形式返回。Flink程序可以运行在不同的环境中，既能够独立运行，也可以嵌入到其他程序中运行。程序可以运行在本地的JVM上，也可以运行在服
    
                    
    • 
                    
  • flink---window
                        搞数据的小杰
flink大数据
                        
    Window介绍DataStream:https://nightlies.apache.org/flink/flink-docs-release-1.17/zh/docs/dev/datastream/operators/windows/SQL:https://nightlies.apache.org/flink/flink-docs-release-1.17/zh/docs/dev/table/
    
                    
    • 
                    
  • Flink(1.13) 的window机制(一)
                        万事万物

                        
    窗口概述在流处理应用中，数据是连续不断的，因此我们不可能等到所有数据都到了才开始处理。当然我们可以每来一个消息就处理一次，但是有时我们需要做一些聚合类的处理，例如：在过去的1分钟内有多少用户点击了我们的网页。在这种情况下，我们必须定义一个窗口，用来收集最近一分钟内的数据，并对这个窗口内的数据进行计算。流式计算是一种被设计用于处理无限数据集的数据处理引擎，而无限数据集是指一种不断增长的本质上无限的数
    
                    
    • 
                    
  • pyflink 自定义函数
                        scan724
Flink实时计算python开发语言
                        
    frompyflink.datastreamimportStreamExecutionEnvironmentfrompyflink.common.typeinfoimportTypesfrompyflink.datastreamimportStreamExecutionEnvironmentfrompyflink.tableimportStreamTableEnvironmentfrompyfli
    
                    
    • 
                    
  • flink 问题记录
                        Jhon_yh
flinkflinkhadoop大数据
                        
    文章目录1.Causedby:java.lang.UnsatisfiedLinkError:org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSums(IILjava/nio/ByteBuffer;ILjava/nio/ByteBuffer;IILjava/lang/String;JZ)V原因java.util.concurrent.Ex
    
                    
    • 
                    
  • Pyflink教程(三)：自定义函数
                        yuxj记录学习
学习笔记学习pyflink
                        
    该文章例子pyflink环境是apache-flink==1.13.6Python自定义函数是PyFlinkTableAPI中最重要的功能之一，其允许用户在PyFlinkTableAPI中使用Python语言开发的自定义函数，极大地拓宽了PythonTableAPI的使用范围。简单来说就是有的业务逻辑和需求是sql语句满足不了或太麻烦的，需要用过函数来实现。PythonUDFPythonUDF，即
    
                    
    • 
                    
  • pyflink 滚动窗口实例
                        菜鸟社长
菜鸟的大数据进阶之路大数据进阶之路kafkabigdatapythonflink
                        
    写在前头：更多大数据相关精彩内容请进我的知识星球，每周定期更新正篇技术路线：模拟kafka生产者发送数据——>flink对kafka数据实时计算处理——>处理后的数据发送到kafka1、模拟客流数据的生产者，参考https://blog.csdn.net/qq_22611181/article/details/1199002502、flink聚合操作原理介绍，参考https://blog.csdn
    
                    
    • 
                                
  • html
                                    周华华
html
                                    
    js 
 1，数组的排列 
var arr=[1,4,234,43,52,]; 
 for(var x=0;x<arr.length;x++){ 
   for(var y=x-1;y<arr.length;y++){ 
     if(arr[x]<arr[y]){ 
    &
    
                                
    • 
                                
  • 【Struts2 四】Struts2拦截器
                                    bit1129
struts2拦截器
                                    
    Struts2框架是基于拦截器实现的，可以对某个Action进行拦截，然后某些逻辑处理，拦截器相当于AOP里面的环绕通知，即在Action方法的执行之前和之后根据需要添加相应的逻辑。事实上，即使struts.xml没有任何关于拦截器的配置，Struts2也会为我们添加一组默认的拦截器，最常见的是，请求参数自动绑定到Action对应的字段上。 
  
Struts2中自定义拦截器的步骤是：
    
                                
    • 
                                
  • make:cc 命令未找到解决方法
                                    daizj
linux命令未知make cc
                                    
    安装rz sz程序时，报下面错误： 
  
[root@slave2 src]# make posix 
cc   -O -DPOSIX -DMD=2 rz.c -o rz 
make: cc：命令未找到 
make: *** [posix] 错误 127 
  
系统：centos 6.6 
环境：虚拟机 
  
错误原因：系统未安装gcc，这个是由于在安
    
                                
    • 
                                
  • Oracle之Job应用
                                    周凡杨
oracle job
                                    
      
最近写服务，服务上线后，需要写一个定时执行的SQL脚本，清理并更新数据库表里的数据，应用到了Oracle 的 Job的相关知识。在此总结一下。 
  
一：查看相关job信息 
   1、相关视图  
 dba_jobs  
 all_jobs  
 user_jobs  
 dba_jobs_running 包含正在运行
    
                                
    • 
                                
  • 多线程机制
                                    朱辉辉33
多线程
                                    
    转至http://blog.csdn.net/lj70024/archive/2010/04/06/5455790.aspx 
程序、进程和线程： 
 程序是一段静态的代码，它是应用程序执行的蓝本。进程是程序的一次动态执行过程，它对应了从代码加载、执行至执行完毕的一个完整过程，这个过程也是进程本身从产生、发展至消亡的过程。线程是比进程更小的单位，一个进程执行过程中可以产生多个线程，每个线程有自身的
    
                                
    • 
                                
  • web报表工具FineReport使用中遇到的常见报错及解决办法（一）
                                    老A不折腾
web报表finereportjava报表报表工具
                                    
    FineReport使用中遇到的常见报错及解决办法（一） 
这里写点抛砖引玉，希望大家能把自己整理的问题及解决方法晾出来，Mark一下，利人利己。 
  
出现问题先搜一下文档上有没有，再看看度娘有没有，再看看论坛有没有。有报错要看日志。下面简单罗列下常见的问题，大多文档上都有提到的。 
  
1、address pool is full： 
含义：地址池满，连接数超过并发数上
    
                                
    • 
                                
  • mysql rpm安装后没有my.cnf
                                    林鹤霄
没有my.cnf
                                    
    Linux下用rpm包安装的MySQL是不会安装/etc/my.cnf文件的， 
至于为什么没有这个文件而MySQL却也能正常启动和作用，在这儿有两个说法， 
第一种说法，my.cnf只是MySQL启动时的一个参数文件，可以没有它，这时MySQL会用内置的默认参数启动， 
第二种说法，MySQL在启动时自动使用/usr/share/mysql目录下的my-medium.cnf文件，这种说法仅限于r
    
                                
    • 
                                
  • Kindle Fire HDX root并安装谷歌服务框架之后仍无法登陆谷歌账号的问题
                                    aigo
root
                                    
    原文：http://kindlefireforkid.com/how-to-setup-a-google-account-on-amazon-fire-tablet/ 
  
Step 4: Run ADB command from your PC 
  
On the PC, you need install Amazon Fire ADB driver and instal
    
                                
    • 
                                
  • javascript 中var提升的典型实例
                                    alxw4616
JavaScript
                                    
    // 刚刚在书上看到的一个小问题,很有意思.大家一起思考下吧
myname = 'global';
var fn = function () {
	console.log(myname); // undefined
	var myname = 'local';
	console.log(myname); // local
};
fn()

// 上述代码实际上等同于以下代码
m
    
                                
    • 
                                
  • 定时器和获取时间的使用
                                    百合不是茶
时间的转换定时器
                                    
    定时器:定时创建任务在游戏设计的时候用的比较多 
  Timer();定时器 
TImerTask();Timer的子类  由 Timer 安排为一次执行或重复执行的任务。 
  
  
  定时器类Timer在java.util包中。使用时，先实例化，然后使用实例的schedule(TimerTask task, long delay)方法，设定
    
                                
    • 
                                
  • JDK1.5 Queue
                                    bijian1013
javathreadjava多线程Queue
                                    
    JDK1.5 Queue 
LinkedList： 
LinkedList不是同步的。如果多个线程同时访问列表，而其中至少一个线程从结构上修改了该列表，则它必须 保持外部同步。（结构修改指添加或删除一个或多个元素的任何操作；仅设置元素的值不是结构修改。）这一般通过对自然封装该列表的对象进行同步操作来完成。如果不存在这样的对象，则应该使用 Collections.synchronizedList 方
    
                                
    • 
                                
  • http认证原理和https
                                    bijian1013
httphttps
                                    
    一.基础介绍 
        在URL前加https://前缀表明是用SSL加密的。 你的电脑与服务器之间收发的信息传输将更加安全。 
        Web服务器启用SSL需要获得一个服务器证书并将该证书与要使用SSL的服务器绑定。 
http和https使用的是完全不同的连接方式，用的端口也不一样,前者是80，后
    
                                
    • 
                                
  • 【Java范型五】范型继承
                                    bit1129
java
                                    
    定义如下一个抽象的范型类，其中定义了两个范型参数，T1，T2 
  
package com.tom.lang.generics;

public abstract class SuperGenerics<T1, T2> {
    private T1 t1;
    private T2 t2;

    public abstract void doIt(T
    
                                
    • 
                                
  • 【Nginx六】nginx.conf常用指令(Directive)
                                    bit1129
Directive
                                    
    1. worker_processes    8; 
表示Nginx将启动8个工作者进程，通过ps -ef|grep nginx,会发现有8个Nginx Worker Process在运行 
  
nobody    53879 118449  0 Apr22 ?        00:26:15 nginx: worker process          
    
                                
    • 
                                
  • lua 遍历Header头部
                                    ronin47
lua header 遍历　
                                    
     
 local headers = ngx.req.get_headers()   
 ngx.say("headers begin", "<br/>")   
 ngx.say("Host : ", he
    
                                
    • 
                                
  • java-32.通过交换a,b中的元素，使[序列a元素的和]与[序列b元素的和]之间的差最小(两数组的差最小)。
                                    bylijinnan
java
                                    
    
import java.util.Arrays;

public class MinSumASumB {

	/**
	 * Q32.有两个序列a,b，大小都为n,序列元素的值任意整数，无序.
	 * 
	 * 要求：通过交换a,b中的元素，使[序列a元素的和]与[序列b元素的和]之间的差最小。 
	 * 例如: 
	 * int[] a = {100,99,98,1,2,3
    
                                
    • 
                                
  • redis
                                    开窍的石头
redis
                                    
    在redis的redis.conf配置文件中找到# requirepass foobared 
把它替换成requirepass 12356789 后边的12356789就是你的密码 
 
打开redis客户端输入config get requirepass 
返回 
redis 127.0.0.1:6379> config get requirepass 
1) "require
    
                                
    • 
                                
  • [JAVA图像与图形]现有的GPU架构支持JAVA语言吗？
                                    comsci
java语言
                                    
     
 
      无论是opengl还是cuda，都是建立在C语言体系架构基础上的，在未来，图像图形处理业务快速发展，相关领域市场不断扩大的情况下，我们JAVA语言系统怎么从这么庞大，且还在不断扩大的市场上分到一块蛋糕，是值得每个JAVAER认真思考和行动的事情 
 
 
       
    
                                
    • 
                                
  • 安装ubuntu14.04登录后花屏了怎么办
                                    cuiyadll
ubuntu
                                    
    这个情况，一般属于显卡驱动问题。 
可以先尝试安装显卡的官方闭源驱动。 
按键盘三个键：CTRL + ALT  +  F1 
进入终端，输入用户名和密码登录终端： 
安装amd的显卡驱动       
sudo  
apt-get  
install  
fglrx       
安装nvidia显卡驱动       
sudo  
ap
    
                                
    • 
                                
  • SSL 与 数字证书 的基本概念和工作原理
                                    darrenzhu
加密ssl证书密钥签名
                                    
    SSL 与 数字证书 的基本概念和工作原理 

http://www.linuxde.net/2012/03/8301.html 
 
SSL握手协议的目的是或最终结果是让客户端和服务器拥有一个共同的密钥，握手协议本身是基于非对称加密机制的，之后就使用共同的密钥基于对称加密机制进行信息交换。 
 
 
 

http://www.ibm.com/developerworks/cn/webspher
    
                                
    • 
                                
  • Ubuntu设置ip的步骤
                                    dcj3sjt126com
ubuntu
                                    
    在单位的一台机器完全装了Ubuntu Server，但回家只能在XP上VM一个，装的时候网卡是DHCP的，用ifconfig查了一下ip是192.168.92.128,可以ping通。 
转载不是错： 
Ubuntu命令行修改网络配置方法 
/etc/network/interfaces打开后里面可设置DHCP或手动设置静态ip。前面auto eth0，让网卡开机自动挂载. 
1. 以D
    
                                
    • 
                                
  • php包管理工具推荐
                                    dcj3sjt126com
PHPComposer
                                    
    http://www.phpcomposer.com/ 
       Composer是 PHP 用来管理依赖（dependency）关系的工具。你可以在自己的项目中声明所依赖的外部工具库（libraries），Composer 会帮你安装这些依赖的库文件。  
中文文档  
入门指南  
下载  
安装包列表   
Composer 中国镜像      
    
                                
    • 
                                
  • Gson使用四（TypeAdapter）
                                    eksliang
jsongsonGson自定义转换器gsonTypeAdapter
                                    
    转载请出自出处：http://eksliang.iteye.com/blog/2175595 一.概述 
       Gson的TypeAapter可以理解成自定义序列化和返序列化 二、应用场景举例 
       例如我们通常去注册时（那些外国网站），会让我们输入firstName，lastName,但是转到我们都
    
                                
    • 
                                
  • JQM控件之Navbar和Tabs
                                    gundumw100
htmlxmlcss
                                    
    在JQM中使用导航栏Navbar是简单的。 
 
只需要将data-role="navbar"赋给div即可： 
 

<div data-role="navbar">
    <ul>
        <li><a href="#" class="ui-btn-active&qu
    
                                
    • 
                                
  • 利用归并排序算法对大文件进行排序
                                    iwindyforest
java归并排序大文件分治法Merge sort
                                    
      
归并排序算法介绍，请参照Wikipeida 
zh.wikipedia.org/wiki/%E5%BD%92%E5%B9%B6%E6%8E%92%E5%BA%8F 
基本思想： 
大文件分割成行数相等的两个子文件，递归（归并排序）两个子文件，直到递归到分割成的子文件低于限制行数 
低于限制行数的子文件直接排序 
两个排序好的子文件归并到父文件 
直到最后所有排序好的父文件归并到输入
    
                                
    • 
                                
  • iOS UIWebView URL拦截
                                    啸笑天
UIWebView
                                    
    本文译者：candeladiao，原文：URL filtering for UIWebView on the iPhone说明：译者在做app开发时，因为页面的javascript文件比较大导致加载速度很慢，所以想把javascript文件打包在app里，当UIWebView需要加载该脚本时就从app本地读取，但UIWebView并不支持加载本地资源。最后从下文中找到了解决方法，第一次翻译，难免有
    
                                
    • 
                                
  • 索引的碎片整理SQL语句
                                    macroli
sql
                                    
    SET NOCOUNT ON
DECLARE @tablename VARCHAR (128)
DECLARE @execstr VARCHAR (255)
DECLARE @objectid INT
DECLARE @indexid INT
DECLARE @frag DECIMAL
DECLARE @maxfrag DECIMAL
--设置最大允许的碎片数量,超过则对索引进行碎片
    
                                
    • 
                                
  • Angularjs同步操作http请求with $promise
                                    qiaolevip
每天进步一点点学习永无止境AngularJS纵观千象
                                    
    // Define a factory
app.factory('profilePromise', ['$q', 'AccountService', function($q, AccountService) {
      var deferred = $q.defer();
      AccountService.getProfile().then(function(res) {
  
    
                                
    • 
                                
  • hibernate联合查询问题
                                    sxj19881213
sqlHibernateHQL联合查询
                                    
    最近在用hibernate做项目，遇到了联合查询的问题，以及联合查询中的N+1问题。 
针对无外键关联的联合查询，我做了HQL和SQL的实验，希望能帮助到大家。（我使用的版本是hibernate3.3.2） 
  
1 几个常识： 
 （1）hql中的几种join查询，只有在外键关联、并且作了相应配置时才能使用。 
 （2）hql的默认查询策略，在进行联合查询时，会产
    
                                
    • 
                                
  • struts2.xml
                                    wuai
struts
                                    
    <?xml version="1.0" encoding="UTF-8" ?> 
<!DOCTYPE struts PUBLIC 
 "-//Apache Software Foundation//DTD Struts Configuration 2.3//EN" 
 "http://struts.apache
    
                                
    •