1.clinet上传200M文件,client根据HDFS配置信息对文件进行拆分block1,block2
2.client 通知NameNode block1,block2存储,副本三份
3-4. NameNode进行权限等校验,根据内部算法合理的计算出block存储的dataNode节点,并将存储信息返回给client
5. client根据NameNode返回的信息先后进行block1,block2文件上传,在上传的过程中,其他两个副本存储是同步进行的,dataNode存储成功会通知nameNode已存储成功,所有的dataNode存储成功之后,nameNode通知client整个写流程结束。
1.Client执行:hadoop fs -get /spark/file.txt 获取文件
2.Client 通知nameNode获取文件的行为,nameNode从原数据获取文件的存储信息block0,,block1信息
3.Client 获取到存储信息直接从dataNode获取,获取成功后自动合并block,下载完成结束。
在程序运行时状态中,通过反射可以知道该类的所有属性和方法;对任意一个对象,通过反射可以调用其属性和方法;通过动态获取信息以及动态获取对象的方法功能称之为反射机制。
// 1 方式1
Class<?> clazz = Class.forName("类全路径")
// 2 方式2
Object obj = new Object();
Class<?> clazz = obj.getClass();
Class<?> clazz = Class.getClass();
略
// 1. 继承Thread
// 2. 实现Runnable
public static main(String[] args) throw Exception {
FetureTask<String> fetureTask = new FetureTask<String>(new Callable<String>() {
// 业务逻辑
return "字符串";
});
new Thread(fetureTask).start();
// fetureTask .get() 方法调用会阻塞,等待线程执行完成
System.out.println(fetureTask .get());
}
synchronized:可以解决线程处理共享数据(共享对象)造成的并发问题。synchronized锁的释放机制:当一个线程获取锁,需要等到线程抛出异常或者线程执行完毕才会释放锁,线程获取锁的期间,其他线程不能操作当前对象。
简单理解:synchronized修饰的方法是对对象加锁,修饰的static方法是对类加锁。
wati:线程等待
notifyAll:唤醒其他所有线程
notify:唤醒其他线程中的一个
注意:notify,notifyAll,notify需要在synchronized关键字中使用,防止异常现象产生
类的装载过程
ClassLoader的loadClass:只完成了第一步,生成class
Class.forName:完成类装载的整个过程
复制算法
优点:无碎片化,简单高效
缺点:可使用的内存变少了(只剩50%),不适合用于老年代对象存储,原因:老年代数据过多,对象内存的重新复制分配排序低效
标记-整理算法
与标记-清除算法类似,不同点:标记-整理算法将被标记为垃圾的对象清除,将存活的对象进行整合顺序排序。此算法解决了:内存碎片化的问题,同时也解决了复制算法内存只能使用50%,但是缺点在处理耗时,适用于对象存活率高的场景:老生代。
默认15次,新生代默认分配的内存占1/3,老年代默认分配的内存2/3。
内存分配模型如:
解决多线程中共享变量的可见性,以及指令防止指令重排序的问题。
https://blog.csdn.net/qq_45376284/article/details/113486716
import com.google.common.collect.Lists;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
public class MyThreadPool {
private final AtomicInteger ctl = new AtomicInteger(0);
// 默认线程池大小
private static final int DEFAULT_THREAD_NUM = 5;
// 执行最大任务
private static final int DEFAULT_MAX_TASK = 20;
// 队列
private BlockingQueue<Runnable> queueRunnable;
// 当前线程池的线程数
private int current_thread_num;
// 当前线程池最大中任务数
private int current_max_task_num;
// 工作的线程
private Set<RunThread> workThread;
static MyThreadPool newFixThreadPool(Integer threadNum, Integer maxTask) {
return new MyThreadPool(threadNum, maxTask);
}
private MyThreadPool(Integer taskNum, Integer maxTask) {
if (Objects.isNull(taskNum) || taskNum <= 0) {
this.current_thread_num = DEFAULT_THREAD_NUM;
} else {
this.current_thread_num = taskNum;
}
if (Objects.isNull(maxTask) || maxTask <= 0) {
this.current_max_task_num = DEFAULT_MAX_TASK;
} else {
this.current_max_task_num = maxTask;
}
this.queueRunnable = new ArrayBlockingQueue<>(this.current_max_task_num);
this.workThread = new HashSet<>();
for (int i = 0; i < current_thread_num; i++) {
RunThread runThread = new RunThread("Thread pool-" + ctl.incrementAndGet());
runThread.start();
this.workThread.add(runThread);
}
}
public void submit(Runnable runnable) {
if (Objects.isNull(runnable)) {
return;
}
queueRunnable.add(runnable);
}
public void submit(List<Runnable> runnableList) {
if (CollectionUtils.isEmpty(runnableList)) {
return;
}
if(runnableList.size() > this.current_max_task_num) {
System.out.println("任务超标");
return;
}
queueRunnable.addAll(runnableList);
}
private final class RunThread extends Thread {
private String threadName;
public RunThread(String name) {
this.threadName = name;
}
@Override
public void run() {
try {
for (;;) {
Runnable runnable = queueRunnable.take();
if (Objects.nonNull(runnable)) {
System.out.println(threadName);
runnable.run();
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
/**
* @describe 测试
* @param args
*/
public static void main(String[] args) {
int num = 20;
List<Runnable> runnableList = Lists.newArrayList();
MyThreadPool myThreadPool = MyThreadPool.newFixThreadPool(2, 20);
for (int i = 0; i < num; i++) {
runnableList.add(new TestThread(i));
}
myThreadPool.submit(runnableList);
}
public static class TestThread implements Runnable {
private int x;
public TestThread(int x) {
this.x = x;
}
@Override
public void run() {
System.out.println("TestThread:" + x);
}
}
}
import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
public class TestThread {
public static void main(String[] args) {
QueueInfo queueInfo = new QueueInfo();
producer producer = new producer(queueInfo);
producer produce1 = new producer(queueInfo);
consumer consumer = new consumer(queueInfo);
consumer consumer1 = new consumer(queueInfo);
consumer consumer2 = new consumer(queueInfo);
producer.start();
consumer.start();
consumer1.start();
consumer2.start();
produce1.start();
}
public static class producer extends Thread {
private QueueInfo queueInfo;
public producer(QueueInfo queueInfo) {
this.queueInfo = queueInfo;
}
@Override
public void run() {
super.run();
ReentrantLock reentrantLock = this.queueInfo.getReentrantLock();
Condition condition = this.queueInfo.getCondition();
// 1. 获取锁 未获取 注释状态
while (true) {
reentrantLock.lock();
try {
// 如果队列满了,阻塞
while (Objects.equals(queueInfo.size(), queueInfo.getMAX())) {
System.out.println(super.getName() + ":队列已满:" + queueInfo.size() + ",等待消费");
Thread.sleep(2000);
condition.await();
}
this.queueInfo.addQueue(super.getName());
Thread.sleep(10);
// 唤醒其他线程
condition.signal();
} catch (InterruptedException e) {
e.printStackTrace();
}finally {
reentrantLock.unlock();
}
}
}
}
public static class consumer extends Thread {
private QueueInfo queueInfo;
public consumer(QueueInfo queueInfo) {
this.queueInfo = queueInfo;
}
@Override
public void run() {
super.run();
ReentrantLock reentrantLock = this.queueInfo.getReentrantLock();
Condition condition = this.queueInfo.getCondition();
// 消费逻辑
// 1. 获取锁 未获取 注释状态
while (true) {
// 加锁消费
reentrantLock.lock();
try {
// 如果队列无消息,通知生产数据
while(CollectionUtils.isEmpty(this.queueInfo)) {
System.out.println(super.getName() + ":已消费完成,等待生产者生产消息");
condition.await();
}
this.queueInfo.poolQueue(super.getName());
Thread.sleep(80);
// 消费完成 唤醒其他生产者或者消费者
condition.signal();
} catch (Exception e) {
e.printStackTrace();
} finally {
// 释放锁
reentrantLock.unlock();
}
}
}
}
}
import java.util.LinkedList;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
public class QueueInfo extends LinkedList<String> {
private final AtomicInteger atomicInteger = new AtomicInteger(0);
private Integer MAX = 20;
// 引入可重入锁
private final ReentrantLock reentrantLock = new ReentrantLock();
private final Condition condition = reentrantLock.newCondition();
public ReentrantLock getReentrantLock() {
return reentrantLock;
}
public Condition getCondition() {
return condition;
}
public void addQueue(String threadName) {
int i = atomicInteger.incrementAndGet();
super.add(i + "");
System.out.println(threadName + ": 生产了一条消息:" + i);
}
public void poolQueue(String threadName) {
String poll = super.poll();
System.out.println(threadName + ": 消费消息:" + poll);
}
public AtomicInteger getAtomicInteger() {
return atomicInteger;
}
public Integer getMAX() {
return MAX;
}
public void setMAX(Integer MAX) {
this.MAX = MAX;
}
}
redis,zookeeper
反射可以获取类的属性,方法,继承关系,可以动态的获取这些信息或者是执行操作。
泛型可以作为类或者是接口存储的对象或者是其描述的对象
线程是一个程序执行过程中所需要的工作者。它由操作系统管理调度所有的线程,将线程分配到任何可用的cpu。线程初始化完成,通过调用run开始执行。当线程执行结束,会被回收。
/**
* public class ListNode {
* int val;
* ListNode next = null;
*
* ListNode(int val) {
* this.val = val;
* }
* }
*
*/
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.Collections;
public class Solution {
List<Integer> list = new ArrayList<Integer>();
public ArrayList<Integer> printListFromTailToHead(ListNode listNode) {
def(listNode);
Collections.reverse(list);
return (ArrayList<Integer>) list;
}
// 递归方法取值
private void def(ListNode listNode) {
if (Objects.nonNull(listNode)) {
list.add(listNode.val);
if(listNode.next != null) {
def(listNode.next);
}
}
}
}
/*
public class ListNode {
int val;
ListNode next = null;
ListNode(int val) {
this.val = val;
}
}*/
public class Solution {
public ListNode ReverseList(ListNode head) {
if(head==null)
return null;
ListNode pre = null;
ListNode next = null;
while(head!=null){
next = head.next;
head.next = pre;
pre = head;
head = next;
}
return pre;
}
}
/*
public class ListNode {
int val;
ListNode next = null;
ListNode(int val) {
this.val = val;
}
}*/
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
public class Solution {
List<ListNode> mergeList = new ArrayList<>();
ListNode resultNode = null;
public ListNode Merge(ListNode list1, ListNode list2) {
List<ListNode> mergeList1 = anz(list1);
List<ListNode> mergeList2 = anz(list2);
mergeList1.addAll(mergeList2);
List<ListNode> mergeList = mergeList1.stream().map(node -> {
node.next = null;
return node;
})
.sorted((n, s) -> {
return n.val - s.val;
}).collect(Collectors.toList());
for (int i = 0; i < mergeList.size(); i++) {
ListNode listNode = mergeList.get(i);
if (i == 0) {
resultNode = listNode;
}
int nex = i + 1;
if (nex < mergeList.size()) {
listNode.next = mergeList.get(nex);
}
}
return resultNode;
}
public List<ListNode> anz(ListNode head) {
List<ListNode> resultList = new ArrayList<>();
if (Objects.isNull(head)) {
return resultList;
}
while (head != null) {
resultList.add(head);
if (head.next == null) {
break;
}
head = head.next;
}
return resultList;
}
}
/*
public class ListNode {
int val;
ListNode next = null;
ListNode(int val) {
this.val = val;
}
}*/
public class Solution {
public ListNode Merge(ListNode list1, ListNode list2) {
ListNode resultNode = new ListNode(0);
ListNode currNode = resultNode;
while (list1 != null || list2 != null) {
// 两个有序得链表,存在空直接返回另一个链表得剩余节点
if(list1 == null) {
currNode.next = list2;
break;
}
if(list2 == null) {
currNode.next = list1;
break;
}
if (list1.val <= list2.val) {
currNode.next = list1;
list1 = list1.next;
} else {
currNode.next = list2;
list2 = list2.next;
}
currNode = currNode.next;
}
return resultNode.next;
}
}
/*
public class ListNode {
int val;
ListNode next = null;
ListNode(int val) {
this.val = val;
}
}*/
public class Solution {
public ListNode FindFirstCommonNode(ListNode pHead1, ListNode pHead2) {
if (pHead1 == null || pHead2 == null) {
return null;
}
ListNode currNode = null;
while (pHead1 != null) {
if (def(pHead1, pHead2)) {
currNode = pHead1;
break;
}
// 下一个
pHead1 = pHead1.next;
}
return currNode;
}
// 递归
private boolean def(ListNode pHead1, ListNode pHead2) {
while (pHead2 != null) {
if (compare(pHead1, pHead2)) {
return true;
}
pHead2 = pHead2.next;
}
return false;
}
public boolean compare(ListNode pHead1, ListNode pHead2) {
while (pHead1 != null && pHead2 != null) {
if (pHead1.val != pHead2.val) {
return false;
}
pHead1 = pHead1.next;
pHead2 = pHead2.next;
}
if (pHead1 == null && pHead2 == null) {
return true;
}
return false;
}
}
import java.util.*;
/*
* public class ListNode {
* int val;
* ListNode next = null;
* public ListNode(int val) {
* this.val = val;
* }
* }
*/
import java.util.ArrayList;
import java.util.List;
public class Solution {
List<ListNode> list = new ArrayList<>();
public ListNode FindKthToTail (ListNode pHead, int k) {
while (pHead != null) {
list.add(pHead);
pHead = pHead.next;
}
// 获取倒数k个
if(k > list.size() || k == 0){
return null;
}
// 1 2 3 4 5 5
return list.get(list.size() - k);
}
}
import java.util.*;
/*
* public class ListNode {
* int val;
* ListNode next = null;
* public ListNode(int val) {
* this.val = val;
* }
* }
*/
import java.util.Stack;
// 压Stack
public class Solution {
Stack<ListNode> stack = new Stack<>();
public ListNode FindKthToTail(ListNode pHead, int k) {
if (k == 0) {
return null;
}
while (pHead != null) {
stack.add(pHead);
pHead = pHead.next;
}
// 获取倒数k个
if (k > stack.size() || k == 0) {
return null;
}
ListNode resNode = null;
while (k > 0) {
k--;
resNode = stack.pop();
}
return resNode;
}
}
/*
public class ListNode {
int val;
ListNode next = null;
ListNode(int val) {
this.val = val;
}
}
*/
import java.util.HashSet;
import java.util.Set;
public class Solution {
Set<Integer> set = new HashSet<>();
public ListNode EntryNodeOfLoop(ListNode pHead) {
while (pHead != null) {
if (!set.add(pHead.val)) {
return pHead;
}
pHead = pHead.next;
}
return null;
}
}
/*
public class RandomListNode {
int label;
RandomListNode next = null;
RandomListNode random = null;
RandomListNode(int label) {
this.label = label;
}
}
*/
import java.util.*;
public class Solution {
RandomListNode root = new RandomListNode(-1);
Map<RandomListNode, RandomListNode> map = new HashMap<>();
public RandomListNode Clone(RandomListNode pHead) {
if (pHead == null) {
return null;
}
RandomListNode currNode = root;
// 1. 先复制单节点
while (pHead != null) {
// 复制节点
RandomListNode newNode = new RandomListNode(pHead.label);
currNode.next = newNode;
currNode = currNode.next;
map.put(pHead, currNode);
pHead = pHead.next;
}
// 2. 节点映射
for (RandomListNode keyNode : map.keySet()) {
RandomListNode randomListNode = map.get(keyNode);
randomListNode.random = map.get(keyNode.random);
}
return root.next;
}
}
/*
public class ListNode {
int val;
ListNode next = null;
ListNode(int val) {
this.val = val;
}
}
*/
import java.util.*;
public class Solution {
Set<Integer> set = new HashSet();
// 存储所有节点 去重
Queue<ListNode> queue1 = new LinkedList<>();
// 存储重复节点
Map<Integer, Boolean> map = new HashMap<>();
public ListNode deleteDuplication(ListNode pHead) {
if (pHead == null) {
return null;
}
while (pHead != null) {
// success 没有重复
if (set.add(pHead.val)) {
queue1.add(pHead);
} else {
map.put(pHead.val, true);
}
pHead = pHead.next;
}
ListNode resultNode = new ListNode(0);
ListNode currNode = resultNode;
for (ListNode node: queue1) {
if(map.containsKey(node.val)) {
continue;
}
node.next = null;
currNode.next = node;
currNode = currNode.next;
}
return resultNode.next;
}
}
/**
public class TreeNode {
int val = 0;
TreeNode left = null;
TreeNode right = null;
public TreeNode(int val) {
this.val = val;
}
}
*/
public class Solution {
public int TreeDepth(TreeNode root) {
if (root == null) {
return 0;
}
return def(root, 1);
}
private int def(TreeNode node, int i) {
// 当前节点为null 或者 左右节点均为null 表示到达最深度
if (node == null|| (node.right == null && node.left == null)) {
return i;
}
i++;
int lef = def(node.left, i);
int rig = def(node.right, i);
return lef > rig ? lef : rig;
}
}
import java.util.ArrayList;
/*
public class TreeNode {
int val = 0;
TreeNode left = null;
TreeNode right = null;
public TreeNode(int val) {
this.val = val;
}
}
*/
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.Queue;
public class Solution {
public ArrayList<ArrayList<Integer>> Print(TreeNode pRoot) {
if(pRoot == null) {
return new ArrayList<>();
}
Queue<TreeNode> q = new LinkedList<>();
ArrayList<ArrayList<Integer>> res = new ArrayList<>();
// 奇数左=>右 偶从右=>左
int index = 1;
q.add(pRoot);
while (!q.isEmpty()) {
// 当前有几个节点
ArrayList<Integer> lineList = new ArrayList<>();
int max = q.size();
for (int i = 0; i < max; i++) {
TreeNode currNode = q.poll();
lineList.add(currNode.val);
// 左到右设置节点
if (currNode.left != null) {
q.add(currNode.left);
}
if (currNode.right != null) {
q.add(currNode.right);
}
}
res.add(lineList);
if (index % 2 == 0) {
Collections.reverse(lineList);
}
index++;
}
return res;
}
}
public class Solution {
public int jumpFloor(int n) {
if(n == 1) {
return 1;
}
if(n == 2) {
return 2;
}
return jumpFloor(n - 1) + jumpFloor(n-2);
}
}
public class Solution {
public int jumpFloorII(int n) {
if(n == 1) {
return 1;
}
return 2*jumpFloorII(n - 1);
}
}