- 问题描述:在使用paddlepaddle启动模型时遇到下列问题:
Traceback (most recent call last):
File "test.py", line 131, in
test(args)
File "test.py", line 77, in test
exe.run(fluid.default_startup_program())
File "/home/spwux/anaconda3/envs/fall/lib/python3.6/site-packages/paddle/fluid/executor.py", line 657, in run
use_program_cache=use_program_cache)
File "/home/spwux/anaconda3/envs/fall/lib/python3.6/site-packages/paddle/fluid/executor.py", line 755, in _run
exe.run(program.desc, scope, 0, True, True, fetch_var_name)
paddle.fluid.core_avx.EnforceNotMet: Invoke operator fill_constant error.
Python Call stacks:
File "/home/spwux/anaconda3/envs/fall/lib/python3.6/site-packages/paddle/fluid/framework.py", line 1845, in _prepend_op
attrs=kwargs.get("attrs", None))
File "/home/spwux/anaconda3/envs/fall/lib/python3.6/site-packages/paddle/fluid/initializer.py", line 189, in __call__
stop_gradient=True)
File "/home/spwux/anaconda3/envs/fall/lib/python3.6/site-packages/paddle/fluid/framework.py", line 1730, in create_parameter
initializer(param, self)
File "/home/spwux/anaconda3/envs/fall/lib/python3.6/site-packages/paddle/fluid/layer_helper_base.py", line 316, in create_parameter
**attr._to_kwargs(with_initializer=True))
File "/home/spwux/anaconda3/envs/fall/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 128, in append_bias_op
attr=bias_attr, shape=size, dtype=input_var.dtype, is_bias=True)
File "/home/spwux/anaconda3/envs/fall/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 2184, in conv2d
pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2)
File "/home/spwux/Desktop/fall/human_pose_estimation/lib/pose_resnet.py", line 110, in net
initializer=fluid.initializer.Normal(0., 0.001)))
File "test.py", line 74, in test
output = model.net(input=image, target=None, target_weight=None)
File "test.py", line 131, in
test(args)
C++ Call stacks:
cudaMemGetInfo failed in paddle::platform::GetMemoryUsage: out of memory at [/paddle/paddle/fluid/platform/gpu_info.cc:215]
PaddlePaddle Call Stacks:
0 0x7fe7d7047cd0p void paddle::platform::EnforceNotMet::Init(char const*, char const*, int) + 352
1 0x7fe7d7048049p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 137
2 0x7fe7d906b032p paddle::platform::GpuMemoryUsage(unsigned long*, unsigned long*) + 146
3 0x7fe7d906b0c3p paddle::platform::GpuAvailableMemToAlloc() + 51
4 0x7fe7d906c3bdp paddle::platform::GpuReallocSize() + 29
5 0x7fe7d906cbcep paddle::platform::GpuMaxAllocSize() + 14
6 0x7fe7d906cc02p paddle::platform::GpuMaxChunkSize() + 18
7 0x7fe7d9051b5ap
8 0x7fe8ea445a99p
9 0x7fe7d90511fdp paddle::memory::legacy::GetGPUBuddyAllocator(int) + 109
10 0x7fe7d9052045p void* paddle::memory::legacy::Alloc(paddle::platform::CUDAPlace const&, unsigned long) + 37
11 0x7fe7d9052585p paddle::memory::allocation::LegacyAllocator::AllocateImpl(unsigned long) + 421
12 0x7fe7d9046635p paddle::memory::allocation::AllocatorFacade::Alloc(boost::variant const&, unsigned long) + 181
13 0x7fe7d90467bap paddle::memory::allocation::AllocatorFacade::AllocShared(boost::variant const&, unsigned long) + 26
14 0x7fe7d8c4368cp paddle::memory::AllocShared(boost::variant const&, unsigned long) + 44
15 0x7fe7d9018e24p paddle::framework::Tensor::mutable_data(boost::variant, paddle::framework::proto::VarType_Type, unsigned long) + 148
16 0x7fe7d741e2aep paddle::operators::FillConstantKernel::Compute(paddle::framework::ExecutionContext const&) const + 494
17 0x7fe7d74213c3p std::_Function_handler, paddle::operators::FillConstantKernel, paddle::operators::FillConstantKernel, paddle::operators::FillConstantKernel, paddle::operators::FillConstantKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35
18 0x7fe7d8fc3a87p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const + 375
19 0x7fe7d8fc3e61p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const + 529
20 0x7fe7d8fbe9f3p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) + 307
21 0x7fe7d71d2d7ep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 398
22 0x7fe7d71d5e1fp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector > const&, bool) + 143
23 0x7fe7d703917dp
24 0x7fe7d707acd6p
25 0x55c7d1a12304p _PyCFunction_FastCallDict + 340
26 0x55c7d1a99c5ep
27 0x55c7d1abc81ap _PyEval_EvalFrameDefault + 778
28 0x55c7d1a92f26p
29 0x55c7d1a93f31p
30 0x55c7d1a99be5p
31 0x55c7d1abd5d9p _PyEval_EvalFrameDefault + 4297
32 0x55c7d1a92f26p
33 0x55c7d1a93f31p
34 0x55c7d1a99be5p
35 0x55c7d1abc81ap _PyEval_EvalFrameDefault + 778
36 0x55c7d1a93cfbp
37 0x55c7d1a99be5p
38 0x55c7d1abc81ap _PyEval_EvalFrameDefault + 778
39 0x55c7d1a94a49p PyEval_EvalCodeEx + 809
40 0x55c7d1a957ecp PyEval_EvalCode + 28
41 0x55c7d1b15e74p
42 0x55c7d1b16271p PyRun_FileExFlags + 161
43 0x55c7d1b16473p PyRun_SimpleFileExFlags + 451
44 0x55c7d1b19f63p Py_Main + 1555
45 0x55c7d19e3f2ep main + 238
46 0x7fe8ea08d830p __libc_start_main + 240
47 0x55c7d1ac327fp
- 问题解决:设置环境变量CUDA_VISIBLE_DEVICES指定使用的显卡
export CUDA_VISIBLE_DEVICES=0